RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into scrapy-check-test

This commit is contained in:
Ajay Mittur 2020-07-08 15:13:37 +05:30 committed by GitHub
parent d014840672 56a6d22352
commit b6d4f28ea7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
303 changed files with 7990 additions and 3988 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.bumpversion.cfg
View File

@ -1,8 +1,7 @@
[bumpversion]
current_version = 1.8.0
current_version = 2.2.0
commit = True
tag = True
tag_name = {new_version}
[bumpversion:file:scrapy/VERSION]

1
.gitignore vendored
View File

@ -15,6 +15,7 @@ htmlcov/
.pytest_cache/
.coverage.*
.cache/
.mypy_cache/
# Windows
Thumbs.db

3
.readthedocs.yml
View File

@ -1,9 +1,12 @@
version: 2
formats: all
sphinx:
configuration: docs/conf.py
fail_on_warning: true
python:
# For available versions, see:
# https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
version: 3.7 # Keep in sync with .travis.yml
install:
- requirements: docs/requirements.txt
- path: .

44
.travis.yml
View File

@ -11,25 +11,35 @@ matrix:
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pypy3
- env: TOXENV=py35
python: 3.5
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=py35-asyncio
python: 3.5.2
- env: TOXENV=py36
python: 3.6
- env: TOXENV=py37
python: 3.7
- env: TOXENV=py38
python: 3.8
- env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=py38-asyncio
- env: TOXENV=pylint
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
- env: TOXENV=typing
python: 3.8
- env: TOXENV=pypy3
- env: TOXENV=pinned
python: 3.5.2
- env: TOXENV=asyncio
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
- env: TOXENV=py
python: 3.5
- env: TOXENV=asyncio
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
- env: TOXENV=py
python: 3.6
- env: TOXENV=py
python: 3.7
- env: TOXENV=py PYPI_RELEASE_JOB=true
python: 3.8
dist: bionic
- env: TOXENV=extra-deps
python: 3.8
dist: bionic
- env: TOXENV=asyncio
python: 3.8
dist: bionic
install:
- |
if [ "$TOXENV" = "pypy3" ]; then
@ -62,4 +72,4 @@ deploy:
on:
tags: true
repo: scrapy/scrapy
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

24
Makefile.buildbot
View File

@ -1,24 +0,0 @@
TRIAL := $(shell which trial)
BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
export PYTHONPATH=$(PWD)
test:
coverage run --branch $(TRIAL) --reporter=text tests
rm -rf htmlcov && coverage html
-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
build:
git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION
debchange -m -D unstable --force-distribution -v \
$$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \
"Automatic build"
debuild -us -uc -b
clean:
git checkout debian scrapy/VERSION
git clean -dfq
pypi:
umask 0022 && chmod -R a+rX . && python setup.py sdist upload
.PHONY: clean test build

4
README.rst
View File

@ -40,8 +40,8 @@ including a list of features.
Requirements
============
* Python 3.5+
* Works on Linux, Windows, Mac OSX, BSD
* Python 3.5.2+
* Works on Linux, Windows, macOS, BSD
Install
=======

3
artwork/README.rst
View File

@ -1,5 +1,4 @@
:orphan:
==============
Scrapy artwork
==============

6
conftest.py
View File

@ -11,7 +11,11 @@ collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess")
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]
for line in open('tests/ignores.txt'):

5
debian/changelog vendored
View File

@ -1,5 +0,0 @@
scrapy (0.11) unstable; urgency=low
* Initial release.
-- Scrapinghub Team <info@scrapinghub.com> Thu, 10 Jun 2010 17:24:02 -0300

1
debian/compat vendored
View File

@ -1 +0,0 @@
7

20
debian/control vendored
View File

@ -1,20 +0,0 @@
Source: scrapy
Section: python
Priority: optional
Maintainer: Scrapinghub Team <info@scrapinghub.com>
Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
Standards-Version: 3.8.4
Homepage: https://scrapy.org/
Package: scrapy
Architecture: all
Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
Recommends: python-setuptools
Conflicts: python-scrapy, scrapy-0.25
Provides: python-scrapy, scrapy-0.25
Description: Python web crawling and web scraping framework
Scrapy is a fast high-level web crawling and web scraping framework,
used to crawl websites and extract structured data from their pages.
It can be used for a wide range of purposes, from data mining to
monitoring and automated testing.

40
debian/copyright vendored
View File

@ -1,40 +0,0 @@
This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
It was downloaded from https://scrapy.org
Upstream Author: Scrapy Developers
Copyright: 2007-2013 Scrapy Developers
License: bsd
Copyright (c) Scrapy developers.
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used
to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The Debian packaging is (C) 2010-2013, Scrapinghub <info@scrapinghub.com> and
is licensed under the BSD, see `/usr/share/common-licenses/BSD'.

1
debian/pyversions vendored
View File

@ -1 +0,0 @@
2.7

5
debian/rules vendored
View File

@ -1,5 +0,0 @@
#!/usr/bin/make -f
# -*- makefile -*-
%:
dh $@

2
debian/scrapy.docs vendored
View File

@ -1,2 +0,0 @@
README.rst
AUTHORS

2
debian/scrapy.install vendored
View File

@ -1,2 +0,0 @@
extras/scrapy_bash_completion etc/bash_completion.d/
extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy

1
debian/scrapy.lintian-overrides vendored
View File

@ -1 +0,0 @@
new-package-should-close-itp-bug

1
debian/scrapy.manpages vendored
View File

@ -1 +0,0 @@
extras/scrapy.1

9
docs/README.rst
View File

@ -57,3 +57,12 @@ There is a way to recreate the doc automatically when you make changes, you
need to install watchdog (``pip install watchdog``) and then use::
make watch
Alternative method using tox
----------------------------
To compile the documentation to HTML run the following command::
tox -e docs
Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.

16
docs/conf.py
View File

@ -1,5 +1,3 @@
# -*- coding: utf-8 -*-
#
# Scrapy documentation build configuration file, created by
# sphinx-quickstart on Mon Nov 24 12:02:52 2008.
#
@ -102,6 +100,9 @@ exclude_trees = ['.build']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# List of Sphinx warnings that will not be raised
suppress_warnings = ['epub.unknown_project_files']
# Options for HTML output
# -----------------------
@ -280,12 +281,15 @@ coverage_ignore_pyobjects = [
# -------------------------------------
intersphinx_mapping = {
'attrs': ('https://www.attrs.org/en/stable/', None),
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
'pytest': ('https://docs.pytest.org/en/latest', None),
'python': ('https://docs.python.org/3', None),
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
'tox': ('https://tox.readthedocs.io/en/latest', None),
'twisted': ('https://twistedmatrix.com/documents/current', None),
'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
}
@ -293,3 +297,11 @@ intersphinx_mapping = {
# ------------------------------------
hoverxref_auto_ref = True
hoverxref_role_types = {
"class": "tooltip",
"confval": "tooltip",
"hoverxref": "tooltip",
"mod": "tooltip",
"ref": "tooltip",
}
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']

11
docs/contributing.rst
View File

@ -143,7 +143,7 @@ by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE``
(replace 'upstream' with a remote name for scrapy repository,
``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE``
with a name of the branch you want to create locally).
See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally.
See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally.
When writing GitHub pull requests, try to keep titles short but descriptive.
E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
@ -155,6 +155,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
removal, etc) in separate commits from functional changes. This will make pull
requests easier to review and more likely to get merged.
.. _coding-style:
Coding style
============
@ -163,12 +166,12 @@ Scrapy:
* Unless otherwise specified, follow :pep:`8`.
* It's OK to use lines longer than 80 chars if it improves the code
* It's OK to use lines longer than 79 chars if it improves the code
readability.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
See https://help.github.com/articles/setting-your-username-in-git/ for
See https://help.github.com/en/github/using-git/setting-your-username-in-git for
setup instructions.
.. _documentation-policies:
@ -266,5 +269,5 @@ And their unit-tests are in::
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
.. _open issues: https://github.com/scrapy/scrapy/issues
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist

40
docs/faq.rst
View File

@ -22,8 +22,8 @@ In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like
comparing `jinja2`_ to `Django`_.
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: http://lxml.de/
.. _jinja2: http://jinja.pocoo.org/
.. _lxml: https://lxml.de/
.. _jinja2: https://palletsprojects.com/p/jinja/
.. _Django: https://www.djangoproject.com/
Can I use Scrapy with BeautifulSoup?
@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
What Python versions does Scrapy support?
-----------------------------------------
Scrapy is supported under Python 3.5+
Scrapy is supported under Python 3.5.2+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
@ -269,7 +269,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For
more info on how it works see `this page`_. Also, here's an `example spider`_
which scrapes one of these sites.
.. _this page: http://search.cpan.org/~ecarroll/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
.. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py
What's the best way to parse big XML/CSV data feeds?
@ -342,15 +342,15 @@ method for this purpose. For example::
from copy import deepcopy
from scrapy.item import BaseItem
from itemadapter import is_item, ItemAdapter
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
for item in result:
if isinstance(item, (BaseItem, dict)):
for _ in range(item['multiply_by']):
if is_item(item):
adapter = ItemAdapter(item)
for _ in range(adapter['multiply_by']):
yield deepcopy(item)
Does Scrapy support IPv6 addresses?
@ -361,6 +361,30 @@ Note that by doing so, you lose the ability to set a specific timeout for DNS re
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
.. _faq-specific-reactor:
How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()`` exceptions?
----------------------------------------------------------------------------------------------
This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
.. _faq-stop-response-download:
How can I cancel the download of a given response?
--------------------------------------------------
In some situations, it might be useful to stop the download of a certain response.
For instance, if you only need the first part of a large response and you would like
to save resources by avoiding the download of the whole body.
In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
the :ref:`topics-stop-response-download` topic for additional information and examples.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search

8
docs/index.rst
View File

@ -165,6 +165,8 @@ Solving specific problems
topics/autothrottle
topics/benchmarking
topics/jobs
topics/coroutines
topics/asyncio
:doc:`faq`
Get answers to most frequently asked questions.
@ -205,6 +207,12 @@ Solving specific problems
:doc:`topics/jobs`
Learn how to pause and resume crawls for large spiders.
:doc:`topics/coroutines`
Use the :ref:`coroutine syntax <async>`.
:doc:`topics/asyncio`
Use :mod:`asyncio` and :mod:`asyncio`-powered libraries.
.. _extending-scrapy:
Extending Scrapy

60
docs/intro/install.rst
View File

@ -7,12 +7,12 @@ Installation guide
Installing Scrapy
=================
Scrapy runs on Python 3.5 or above
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Scrapy runs on Python 3.5.2 or above under CPython (default Python
implementation) and PyPy (starting with PyPy 5.9).
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
and OS X.
and macOS.
To install Scrapy using ``conda``, run::
@ -65,7 +65,7 @@ please refer to their respective installation instructions:
* `lxml installation`_
* `cryptography installation`_
.. _lxml installation: http://lxml.de/installation.html
.. _lxml installation: https://lxml.de/installation.html
.. _cryptography installation: https://cryptography.io/en/latest/installation/
@ -81,35 +81,18 @@ Python packages can be installed either globally (a.k.a system wide),
or in user-space. We do not recommend installing Scrapy system wide.
Instead, we recommend that you install Scrapy within a so-called
"virtual environment" (`virtualenv`_).
Virtualenvs allow you to not conflict with already-installed Python
"virtual environment" (:mod:`venv`).
Virtual environments allow you to not conflict with already-installed Python
system packages (which could break some of your system tools and scripts),
and still install packages normally with ``pip`` (without ``sudo`` and the likes).
To get started with virtual environments, see `virtualenv installation instructions`_.
To install it globally (having it globally installed actually helps here),
it should be a matter of running::
See :ref:`tut-venv` on how to create your virtual environment.
$ [sudo] pip install virtualenv
Check this `user guide`_ on how to create your virtualenv.
.. note::
If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
Once you have created a virtualenv, you can install Scrapy inside it with ``pip``,
Once you have created a virtual environment, you can install Scrapy inside it with ``pip``,
just like any other Python package.
(See :ref:`platform-specific guides <intro-install-platform-notes>`
below for non-Python dependencies that you may need to install beforehand).
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
only supports Python 3, make sure you created a Python 3 virtualenv.
.. _virtualenv: https://virtualenv.pypa.io
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html
.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
.. _intro-install-platform-notes:
@ -165,11 +148,11 @@ you can install Scrapy with ``pip`` after that::
.. _intro-install-macos:
Mac OS X
--------
macOS
-----
Building Scrapy's dependencies requires the presence of a C compiler and
development headers. On OS X this is typically provided by Apple’s Xcode
development headers. On macOS this is typically provided by Apple’s Xcode
development tools. To install the Xcode command line tools open a terminal
window and run::
@ -205,15 +188,12 @@ solutions:
brew update; brew upgrade python
* *(Optional)* Install Scrapy inside an isolated python environment.
* *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
<intro-using-virtualenv>`.
This method is a workaround for the above OS X issue, but it's an overall
This method is a workaround for the above macOS issue, but it's an overall
good practice for managing dependencies and can complement the first method.
`virtualenv`_ is a tool you can use to create virtual environments in python.
We recommended reading a tutorial like
http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started.
After any of these workarounds you should be able to install Scrapy::
pip install Scrapy
@ -227,7 +207,7 @@ For PyPy3, only Linux installation was tested.
Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
On OS X, you are likely to face an issue with building Cryptography dependency,
On macOS, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
that is to ``brew install openssl`` and then export the flags that this command
@ -273,11 +253,11 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
.. _Python: https://www.python.org/
.. _pip: https://pip.pypa.io/en/latest/installing/
.. _lxml: https://lxml.de/index.html
.. _parsel: https://pypi.python.org/pypi/parsel
.. _w3lib: https://pypi.python.org/pypi/w3lib
.. _twisted: https://twistedmatrix.com/
.. _cryptography: https://cryptography.io/
.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
.. _parsel: https://pypi.org/project/parsel/
.. _w3lib: https://pypi.org/project/w3lib/
.. _twisted: https://twistedmatrix.com/trac/
.. _cryptography: https://cryptography.io/en/latest/
.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
.. _setuptools: https://pypi.python.org/pypi/setuptools
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
.. _homebrew: https://brew.sh/

21
docs/intro/tutorial.rst
View File

@ -25,16 +25,16 @@ Scrapy.
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
If you're new to programming and want to start with Python, the following books
may be useful to you:
may be useful to you:
* `Automate the Boring Stuff With Python`_
* `How To Think Like a Computer Scientist`_
* `How To Think Like a Computer Scientist`_
* `Learn Python 3 The Hard Way`_
* `Learn Python 3 The Hard Way`_
You can also take a look at `this list of Python resources for non-programmers`_,
as well as the `suggested resources in the learnpython-subreddit`_.
as well as the `suggested resources in the learnpython-subreddit`_.
.. _Python: https://www.python.org/
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents::
__init__.py
items.py # project items definition file
middlewares.py # project middlewares file
pipelines.py # project pipelines file
@ -212,7 +212,7 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
.. note::
Remember to always enclose urls in quotes when running Scrapy shell from
command-line, otherwise urls containing arguments (ie. ``&`` character)
command-line, otherwise urls containing arguments (i.e. ``&`` character)
will not work.
On Windows, use double quotes instead::
@ -287,8 +287,8 @@ to be scraped, you can at least get **some** data.
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
expressions`_:
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
:doc:`regular expressions <library/re>`:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
@ -305,8 +305,7 @@ with a selector (see :ref:`topics-developer-tools`).
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
visually selected elements, which works in many browsers.
.. _regular expressions: https://docs.python.org/3/library/re.html
.. _Selector Gadget: http://selectorgadget.com/
.. _Selector Gadget: https://selectorgadget.com/
XPath: a brief intro
@ -337,7 +336,7 @@ recommend `this tutorial to learn XPath through examples
<http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
.. _XPath: https://www.w3.org/TR/xpath
.. _XPath: https://www.w3.org/TR/xpath/all/
.. _CSS: https://www.w3.org/TR/selectors
Extracting quotes and authors

844
docs/news.rst
View File

@ -3,8 +3,810 @@
Release notes
=============
.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
.. _release-2.2.0:
Scrapy 2.2.0 (2020-06-24)
-------------------------
Highlights:
* Python 3.5.2+ is required now
* :ref:`dataclass objects <dataclass-items>` and
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
<item-types>`
* New :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
* New :signal:`bytes_received` signal that allows canceling response download
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to
run with a Python version lower than 3.5.2, which introduced
:class:`typing.Type` (:issue:`4615`)
Deprecations
~~~~~~~~~~~~
* :meth:`TextResponse.body_as_unicode
<scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use
:attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead
(:issue:`4546`, :issue:`4555`, :issue:`4579`)
* :class:`scrapy.item.BaseItem` is now deprecated, use
:class:`scrapy.item.Item` instead (:issue:`4534`)
New features
~~~~~~~~~~~~
* :ref:`dataclass objects <dataclass-items>` and
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
<item-types>`, and a new itemadapter_ library makes it easy to
write code that :ref:`supports any item type <supporting-item-types>`
(:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`)
* A new :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`,
:issue:`4574`)
* A new :signal:`bytes_received` signal allows monitoring response download
progress and :ref:`stopping downloads <topics-stop-response-download>`
(:issue:`4205`, :issue:`4559`)
* The dictionaries in the result list of a :ref:`media pipeline
<topics-media-pipeline>` now include a new key, ``status``, which indicates
if the file was downloaded or, if the file was not downloaded, why it was
not downloaded; see :meth:`FilesPipeline.get_media_requests
<scrapy.pipelines.files.FilesPipeline.get_media_requests>` for more
information (:issue:`2893`, :issue:`4486`)
* When using :ref:`Google Cloud Storage <media-pipeline-gcs>` for
a :ref:`media pipeline <topics-media-pipeline>`, a warning is now logged if
the configured credentials do not grant the required permissions
(:issue:`4346`, :issue:`4508`)
* :ref:`Link extractors <topics-link-extractors>` are now serializable,
as long as you do not use :ref:`lambdas <lambda>` for parameters; for
example, you can now pass link extractors in :attr:`Request.cb_kwargs
<scrapy.http.Request.cb_kwargs>` or
:attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting
scheduled requests <topics-jobs>` (:issue:`4554`)
* Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses
from protocol 2 to protocol 4, improving serialization capabilities and
performance (:issue:`4135`, :issue:`4541`)
* :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError`
exception if the resulting instance is ``None`` (:issue:`4528`,
:issue:`4532`)
.. _itemadapter: https://github.com/scrapy/itemadapter
Bug fixes
~~~~~~~~~
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
discards cookies defined in :attr:`Request.headers
<scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`)
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter
of the ``__init__`` method of :class:`~scrapy.http.Request`
(:issue:`2400`, :issue:`3575`)
* When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is
``False`` and the crawl yields no items, Scrapy no longer stops feed
exports after the first URI (:issue:`4621`, :issue:`4626`)
* :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine
syntax <topics/coroutines>` no longer need to return an iterable, and may
instead return a :class:`~scrapy.http.Request` object, an
:ref:`item <topics-items>`, or ``None`` (:issue:`4609`)
* The :command:`startproject` command now ensures that the generated project
folders and files have the right permissions (:issue:`4604`)
* Fix a :exc:`KeyError` exception being sometimes raised from
:class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`,
:issue:`4599`)
* When :setting:`FEEDS` defines multiple URIs, log messages about items being
stored now contain information from the corresponding feed, instead of
always containing information about only one of the feeds (:issue:`4619`,
:issue:`4629`)
Documentation
~~~~~~~~~~~~~
* Added a new section about :ref:`accessing cb_kwargs from errbacks
<errback-cb_kwargs>` (:issue:`4598`, :issue:`4634`)
* Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`,
:issue:`4562`)
* Removed from :doc:`topics/coroutines` the warning about the API being
experimental (:issue:`4511`, :issue:`4513`)
* Removed references to unsupported versions of :doc:`Twisted
<twisted:index>` (:issue:`4533`)
* Updated the description of the :ref:`screenshot pipeline example
<ScreenshotPipeline>`, which now uses :doc:`coroutine syntax
<topics/coroutines>` instead of returning a
:class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`)
* Removed a misleading import line from the
:func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`,
:issue:`4587`)
* The display-on-hover behavior of internal documentation references now also
covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta
<scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and
:ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`)
* It is again possible to download the documentation for offline reading
(:issue:`4578`, :issue:`4585`)
* Removed backslashes preceding ``*args`` and ``**kwargs`` in some function
and method signatures (:issue:`4592`, :issue:`4596`)
.. _chompjs: https://github.com/Nykakin/chompjs
Quality assurance
~~~~~~~~~~~~~~~~~
* Adjusted the code base further to our :ref:`style guidelines
<coding-style>` (:issue:`4237`, :issue:`4525`, :issue:`4538`,
:issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`,
:issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`,
:issue:`4572`)
* Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`,
:issue:`4568`)
* Improved code sharing between the :command:`crawl` and :command:`runspider`
commands (:issue:`4548`, :issue:`4552`)
* Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)``
(:issue:`4635`)
* You may now run the :mod:`asyncio` tests with Tox on any Python version
(:issue:`4521`)
* Updated test requirements to reflect an incompatibility with pytest 5.4 and
5.4.1 (:issue:`4588`)
* Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for
scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`)
* Configured Travis CI to also run the tests with Python 3.5.2
(:issue:`4518`, :issue:`4615`)
* Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI
(:issue:`3727`)
* Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`)
* Made use of set literals in tests (:issue:`4573`)
* Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`,
:issue:`4522`, :issue:`4537`)
.. _release-2.1.0:
Scrapy 2.1.0 (2020-04-24)
-------------------------
Highlights:
* New :setting:`FEEDS` setting to export to multiple feeds
* New :attr:`Response.ip_address <scrapy.http.Response.ip_address>` attribute
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* :exc:`AssertionError` exceptions triggered by :ref:`assert <assert>`
statements have been replaced by new exception types, to support running
Python in optimized mode (see :option:`-O`) without changing Scrapy’s
behavior in any unexpected ways.
If you catch an :exc:`AssertionError` exception from Scrapy, update your
code to catch the corresponding new exception.
(:issue:`4440`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use
:setting:`SCHEDULER_DEBUG` instead (:issue:`4385`)
* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use
:setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`)
* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware`
middleware has been removed, including the entire
:class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers
work out of the box (:issue:`4431`)
* The ``spiders`` property has been removed from
:class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader
<scrapy.crawler.CrawlerRunner.spider_loader>` or instantiate
:setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`)
* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode``
classes have been removed from :mod:`scrapy.utils.datatypes`
(:issue:`4400`)
Deprecations
~~~~~~~~~~~~
* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in
favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`,
:issue:`4507`)
New features
~~~~~~~~~~~~
* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds
with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The :command:`crawl` and :command:`runspider` commands now support multiple
``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The :command:`crawl` and :command:`runspider` commands now support
specifying an output format by appending ``:<format>`` to the output file
(:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The new :attr:`Response.ip_address <scrapy.http.Response.ip_address>`
attribute gives access to the IP address that originated a response
(:issue:`3903`, :issue:`3940`)
* A warning is now issued when a value in
:attr:`~scrapy.spiders.Spider.allowed_domains` includes a port
(:issue:`50`, :issue:`3198`, :issue:`4413`)
* Zsh completion now excludes used option aliases from the completion list
(:issue:`4438`)
Bug fixes
~~~~~~~~~
* :ref:`Request serialization <request-serialization>` no longer breaks for
callbacks that are spider attributes which are assigned a function with a
different name (:issue:`4500`)
* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer
cause a :exc:`TypeError` exception (:issue:`4410`)
* Zsh completion no longer allows options after arguments (:issue:`4438`)
* zope.interface 5.0.0 and later versions are now supported
(:issue:`4447`, :issue:`4448`)
* :meth:`Spider.make_requests_from_url
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
1.4.0, now issues a warning when used (:issue:`4412`)
Documentation
~~~~~~~~~~~~~
* Improved the documentation about signals that allow their handlers to
return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`,
:issue:`4390`)
* Our PyPI entry now includes links for our documentation, our source code
repository and our issue tracker (:issue:`4456`)
* Covered the `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_
service in the documentation (:issue:`4206`, :issue:`4455`)
* Removed references to the Guppy library, which only works in Python 2
(:issue:`4285`, :issue:`4343`)
* Extended use of InterSphinx to link to Python 3 documentation
(:issue:`4444`, :issue:`4445`)
* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`,
:issue:`4496`, :issue:`4503`)
Quality assurance
~~~~~~~~~~~~~~~~~
* Removed warnings about using old, removed settings (:issue:`4404`)
* Removed a warning about importing
:class:`~twisted.internet.testing.StringTransport` from
``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`)
* Removed outdated Debian package build files (:issue:`4384`)
* Removed :class:`object` usage as a base class (:issue:`4430`)
* Removed code that added support for old versions of Twisted that we no
longer support (:issue:`4472`)
* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`,
:issue:`4481`)
* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`,
:issue:`4446`, :issue:`4489`)
.. _release-2.0.1:
Scrapy 2.0.1 (2020-03-18)
-------------------------
* :meth:`Response.follow_all <scrapy.http.Response.follow_all>` now supports
an empty URL iterable as input (:issue:`4408`, :issue:`4420`)
* Removed top-level :mod:`~twisted.internet.reactor` imports to prevent
errors about the wrong Twisted reactor being installed when setting a
different Twisted reactor using :setting:`TWISTED_REACTOR` (:issue:`4401`,
:issue:`4406`)
* Fixed tests (:issue:`4422`)
.. _release-2.0.0:
Scrapy 2.0.0 (2020-03-03)
-------------------------
Highlights:
* Python 2 support has been removed
* :doc:`Partial <topics/coroutines>` :ref:`coroutine syntax <async>` support
and :doc:`experimental <topics/asyncio>` :mod:`asyncio` support
* New :meth:`Response.follow_all <scrapy.http.Response.follow_all>` method
* :ref:`FTP support <media-pipeline-ftp>` for media pipelines
* New :attr:`Response.certificate <scrapy.http.Response.certificate>`
attribute
* IPv6 support through :setting:`DNS_RESOLVER`
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Python 2 support has been removed, following `Python 2 end-of-life on
January 1, 2020`_ (:issue:`4091`, :issue:`4114`, :issue:`4115`,
:issue:`4121`, :issue:`4138`, :issue:`4231`, :issue:`4242`, :issue:`4304`,
:issue:`4309`, :issue:`4373`)
* Retry gaveups (see :setting:`RETRY_TIMES`) are now logged as errors instead
of as debug information (:issue:`3171`, :issue:`3566`)
* File extensions that
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
ignores by default now also include ``7z``, ``7zip``, ``apk``, ``bz2``,
``cdr``, ``dmg``, ``ico``, ``iso``, ``tar``, ``tar.gz``, ``webm``, and
``xz`` (:issue:`1837`, :issue:`2067`, :issue:`4066`)
* The :setting:`METAREFRESH_IGNORE_TAGS` setting is now an empty list by
default, following web browser behavior (:issue:`3844`, :issue:`4311`)
* The
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
now includes spaces after commas in the value of the ``Accept-Encoding``
header that it sets, following web browser behavior (:issue:`4293`)
* The ``__init__`` method of custom download handlers (see
:setting:`DOWNLOAD_HANDLERS`) or subclasses of the following downloader
handlers no longer receives a ``settings`` parameter:
* :class:`scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler`
* :class:`scrapy.core.downloader.handlers.file.FileDownloadHandler`
Use the ``from_settings`` or ``from_crawler`` class methods to expose such
a parameter to your custom download handlers.
(:issue:`4126`)
* We have refactored the :class:`scrapy.core.scheduler.Scheduler` class and
related queue classes (see :setting:`SCHEDULER_PRIORITY_QUEUE`,
:setting:`SCHEDULER_DISK_QUEUE` and :setting:`SCHEDULER_MEMORY_QUEUE`) to
make it easier to implement custom scheduler queue classes. See
:ref:`2-0-0-scheduler-queue-changes` below for details.
* Overridden settings are now logged in a different format. This is more in
line with similar information logged at startup (:issue:`4199`)
.. _Python 2 end-of-life on January 1, 2020: https://www.python.org/doc/sunset-python-2/
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* The :ref:`Scrapy shell <topics-shell>` no longer provides a `sel` proxy
object, use :meth:`response.selector <scrapy.http.Response.selector>`
instead (:issue:`4347`)
* LevelDB support has been removed (:issue:`4112`)
* The following functions have been removed from :mod:`scrapy.utils.python`:
``isbinarytext``, ``is_writable``, ``setattr_default``, ``stringify_dict``
(:issue:`4362`)
Deprecations
~~~~~~~~~~~~
* Using environment variables prefixed with ``SCRAPY_`` to override settings
is deprecated (:issue:`4300`, :issue:`4374`, :issue:`4375`)
* :class:`scrapy.linkextractors.FilteringLinkExtractor` is deprecated, use
:class:`scrapy.linkextractors.LinkExtractor
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` instead (:issue:`4045`)
* The ``noconnect`` query string argument of proxy URLs is deprecated and
should be removed from proxy URLs (:issue:`4198`)
* The :meth:`next <scrapy.utils.python.MutableChain.next>` method of
:class:`scrapy.utils.python.MutableChain` is deprecated, use the global
:func:`next` function or :meth:`MutableChain.__next__
<scrapy.utils.python.MutableChain.__next__>` instead (:issue:`4153`)
New features
~~~~~~~~~~~~
* Added :doc:`partial support <topics/coroutines>` for Python’s
:ref:`coroutine syntax <async>` and :doc:`experimental support
<topics/asyncio>` for :mod:`asyncio` and :mod:`asyncio`-powered libraries
(:issue:`4010`, :issue:`4259`, :issue:`4269`, :issue:`4270`, :issue:`4271`,
:issue:`4316`, :issue:`4318`)
* The new :meth:`Response.follow_all <scrapy.http.Response.follow_all>`
method offers the same functionality as
:meth:`Response.follow <scrapy.http.Response.follow>` but supports an
iterable of URLs as input and returns an iterable of requests
(:issue:`2582`, :issue:`4057`, :issue:`4286`)
* :ref:`Media pipelines <topics-media-pipeline>` now support :ref:`FTP
storage <media-pipeline-ftp>` (:issue:`3928`, :issue:`3961`)
* The new :attr:`Response.certificate <scrapy.http.Response.certificate>`
attribute exposes the SSL certificate of the server as a
:class:`twisted.internet.ssl.Certificate` object for HTTPS responses
(:issue:`2726`, :issue:`4054`)
* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support
(:issue:`1031`, :issue:`4227`)
* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring
the existing soft limit that pauses request downloads when the total
response data being processed is too high (:issue:`1410`, :issue:`3551`)
* A new :setting:`TWISTED_REACTOR` setting allows customizing the
:mod:`~twisted.internet.reactor` that Scrapy uses, allowing to
:doc:`enable asyncio support <topics/asyncio>` or deal with a
:ref:`common macOS issue <faq-specific-reactor>` (:issue:`2905`,
:issue:`4294`)
* Scheduler disk and memory queues may now use the class methods
``from_crawler`` or ``from_settings`` (:issue:`3884`)
* The new :attr:`Response.cb_kwargs <scrapy.http.Response.cb_kwargs>`
attribute serves as a shortcut for :attr:`Response.request.cb_kwargs
<scrapy.http.Request.cb_kwargs>` (:issue:`4331`)
* :meth:`Response.follow <scrapy.http.Response.follow>` now supports a
``flags`` parameter, for consistency with :class:`~scrapy.http.Request`
(:issue:`4277`, :issue:`4279`)
* :ref:`Item loader processors <topics-loaders-processors>` can now be
regular functions, they no longer need to be methods (:issue:`3899`)
* :class:`~scrapy.spiders.Rule` now accepts an ``errback`` parameter
(:issue:`4000`)
* :class:`~scrapy.http.Request` no longer requires a ``callback`` parameter
when an ``errback`` parameter is specified (:issue:`3586`, :issue:`4008`)
* :class:`~scrapy.logformatter.LogFormatter` now supports some additional
methods:
* :class:`~scrapy.logformatter.LogFormatter.download_error` for
download errors
* :class:`~scrapy.logformatter.LogFormatter.item_error` for exceptions
raised during item processing by :ref:`item pipelines
<topics-item-pipeline>`
* :class:`~scrapy.logformatter.LogFormatter.spider_error` for exceptions
raised from :ref:`spider callbacks <topics-spiders>`
(:issue:`374`, :issue:`3986`, :issue:`3989`, :issue:`4176`, :issue:`4188`)
* The :setting:`FEED_URI` setting now supports :class:`pathlib.Path` values
(:issue:`3731`, :issue:`4074`)
* A new :signal:`request_left_downloader` signal is sent when a request
leaves the downloader (:issue:`4303`)
* Scrapy logs a warning when it detects a request callback or errback that
uses ``yield`` but also returns a value, since the returned value would be
lost (:issue:`3484`, :issue:`3869`)
* :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError`
exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls`
attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`,
but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`)
* :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use
``super().__init__(**kwargs)`` instead of ``self._configure(kwargs)`` in
their ``__init__`` method, passing ``dont_fail=True`` to the parent
``__init__`` method if needed, and accessing ``kwargs`` at ``self._kwargs``
after calling their parent ``__init__`` method (:issue:`4193`,
:issue:`4370`)
* A new ``keep_fragments`` parameter of
:func:`scrapy.utils.request.request_fingerprint` allows to generate
different fingerprints for requests with different fragments in their URL
(:issue:`4104`)
* Download handlers (see :setting:`DOWNLOAD_HANDLERS`) may now use the
``from_settings`` and ``from_crawler`` class methods that other Scrapy
components already supported (:issue:`4126`)
* :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``,
`allowing it to be used as a sequence <https://lgtm.com/rules/4850080/>`_
(:issue:`4153`)
Bug fixes
~~~~~~~~~
* The :command:`crawl` command now also exits with exit code 1 when an
exception happens before the crawling starts (:issue:`4175`, :issue:`4207`)
* :class:`LinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` no longer
re-encodes the query string or URLs from non-UTF-8 responses in UTF-8
(:issue:`998`, :issue:`1403`, :issue:`1949`, :issue:`4321`)
* The first spider middleware (see :setting:`SPIDER_MIDDLEWARES`) now also
processes exceptions raised from callbacks that are generators
(:issue:`4260`, :issue:`4272`)
* Redirects to URLs starting with 3 slashes (``///``) are now supported
(:issue:`4032`, :issue:`4042`)
* :class:`~scrapy.http.Request` no longer accepts strings as ``url`` simply
because they have a colon (:issue:`2552`, :issue:`4094`)
* The correct encoding is now used for attach names in
:class:`~scrapy.mail.MailSender` (:issue:`4229`, :issue:`4239`)
* :class:`~scrapy.dupefilters.RFPDupeFilter`, the default
:setting:`DUPEFILTER_CLASS`, no longer writes an extra ``\r`` character on
each line in Windows, which made the size of the ``requests.seen`` file
unnecessarily large on that platform (:issue:`4283`)
* Z shell auto-completion now looks for ``.html`` files, not ``.http`` files,
and covers the ``-h`` command-line switch (:issue:`4122`, :issue:`4291`)
* Adding items to a :class:`scrapy.utils.datatypes.LocalCache` object
without a ``limit`` defined no longer raises a :exc:`TypeError` exception
(:issue:`4123`)
* Fixed a typo in the message of the :exc:`ValueError` exception raised when
:func:`scrapy.utils.misc.create_instance` gets both ``settings`` and
``crawler`` set to ``None`` (:issue:`4128`)
Documentation
~~~~~~~~~~~~~
* API documentation now links to an online, syntax-highlighted view of the
corresponding source code (:issue:`4148`)
* Links to unexisting documentation pages now allow access to the sidebar
(:issue:`4152`, :issue:`4169`)
* Cross-references within our documentation now display a tooltip when
hovered (:issue:`4173`, :issue:`4183`)
* Improved the documentation about :meth:`LinkExtractor.extract_links
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` and
simplified :ref:`topics-link-extractors` (:issue:`4045`)
* Clarified how :class:`ItemLoader.item <scrapy.loader.ItemLoader.item>`
works (:issue:`3574`, :issue:`4099`)
* Clarified that :func:`logging.basicConfig` should not be used when also
using :class:`~scrapy.crawler.CrawlerProcess` (:issue:`2149`,
:issue:`2352`, :issue:`3146`, :issue:`3960`)
* Clarified the requirements for :class:`~scrapy.http.Request` objects
:ref:`when using persistence <request-serialization>` (:issue:`4124`,
:issue:`4139`)
* Clarified how to install a :ref:`custom image pipeline
<media-pipeline-example>` (:issue:`4034`, :issue:`4252`)
* Fixed the signatures of the ``file_path`` method in :ref:`media pipeline
<topics-media-pipeline>` examples (:issue:`4290`)
* Covered a backward-incompatible change in Scrapy 1.7.0 affecting custom
:class:`scrapy.core.scheduler.Scheduler` subclasses (:issue:`4274`)
* Improved the ``README.rst`` and ``CODE_OF_CONDUCT.md`` files
(:issue:`4059`)
* Documentation examples are now checked as part of our test suite and we
have fixed some of the issues detected (:issue:`4142`, :issue:`4146`,
:issue:`4171`, :issue:`4184`, :issue:`4190`)
* Fixed logic issues, broken links and typos (:issue:`4247`, :issue:`4258`,
:issue:`4282`, :issue:`4288`, :issue:`4305`, :issue:`4308`, :issue:`4323`,
:issue:`4338`, :issue:`4359`, :issue:`4361`)
* Improved consistency when referring to the ``__init__`` method of an object
(:issue:`4086`, :issue:`4088`)
* Fixed an inconsistency between code and output in :ref:`intro-overview`
(:issue:`4213`)
* Extended :mod:`~sphinx.ext.intersphinx` usage (:issue:`4147`,
:issue:`4172`, :issue:`4185`, :issue:`4194`, :issue:`4197`)
* We now use a recent version of Python to build the documentation
(:issue:`4140`, :issue:`4249`)
* Cleaned up documentation (:issue:`4143`, :issue:`4275`)
Quality assurance
~~~~~~~~~~~~~~~~~
* Re-enabled proxy ``CONNECT`` tests (:issue:`2545`, :issue:`4114`)
* Added Bandit_ security checks to our test suite (:issue:`4162`,
:issue:`4181`)
* Added Flake8_ style checks to our test suite and applied many of the
corresponding changes (:issue:`3944`, :issue:`3945`, :issue:`4137`,
:issue:`4157`, :issue:`4167`, :issue:`4174`, :issue:`4186`, :issue:`4195`,
:issue:`4238`, :issue:`4246`, :issue:`4355`, :issue:`4360`, :issue:`4365`)
* Improved test coverage (:issue:`4097`, :issue:`4218`, :issue:`4236`)
* Started reporting slowest tests, and improved the performance of some of
them (:issue:`4163`, :issue:`4164`)
* Fixed broken tests and refactored some tests (:issue:`4014`, :issue:`4095`,
:issue:`4244`, :issue:`4268`, :issue:`4372`)
* Modified the :doc:`tox <tox:index>` configuration to allow running tests
with any Python version, run Bandit_ and Flake8_ tests by default, and
enforce a minimum tox version programmatically (:issue:`4179`)
* Cleaned up code (:issue:`3937`, :issue:`4208`, :issue:`4209`,
:issue:`4210`, :issue:`4212`, :issue:`4369`, :issue:`4376`, :issue:`4378`)
.. _Bandit: https://bandit.readthedocs.io/
.. _Flake8: https://flake8.pycqa.org/en/latest/
.. _2-0-0-scheduler-queue-changes:
Changes to scheduler queue classes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The following changes may impact any custom queue classes of all types:
* The ``push`` method no longer receives a second positional parameter
containing ``request.priority * -1``. If you need that value, get it
from the first positional parameter, ``request``, instead, or use
the new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority`
method in :class:`scrapy.core.scheduler.ScrapyPriorityQueue`
subclasses.
The following changes may impact custom priority queue classes:
* In the ``__init__`` method or the ``from_crawler`` or ``from_settings``
class methods:
* The parameter that used to contain a factory function,
``qfactory``, is now passed as a keyword parameter named
``downstream_queue_cls``.
* A new keyword parameter has been added: ``key``. It is a string
that is always an empty string for memory queues and indicates the
:setting:`JOB_DIR` value for disk queues.
* The parameter for disk queues that contains data from the previous
crawl, ``startprios`` or ``slot_startprios``, is now passed as a
keyword parameter named ``startprios``.
* The ``serialize`` parameter is no longer passed. The disk queue
class must take care of request serialization on its own before
writing to disk, using the
:func:`~scrapy.utils.reqser.request_to_dict` and
:func:`~scrapy.utils.reqser.request_from_dict` functions from the
:mod:`scrapy.utils.reqser` module.
The following changes may impact custom disk and memory queue classes:
* The signature of the ``__init__`` method is now
``__init__(self, crawler, key)``.
The following changes affect specifically the
:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` and
:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` classes from
:mod:`scrapy.core.scheduler` and may affect subclasses:
* In the ``__init__`` method, most of the changes described above apply.
``__init__`` may still receive all parameters as positional parameters,
however:
* ``downstream_queue_cls``, which replaced ``qfactory``, must be
instantiated differently.
``qfactory`` was instantiated with a priority value (integer).
Instances of ``downstream_queue_cls`` should be created using
the new
:meth:`ScrapyPriorityQueue.qfactory <scrapy.core.scheduler.ScrapyPriorityQueue.qfactory>`
or
:meth:`DownloaderAwarePriorityQueue.pqfactory <scrapy.core.scheduler.DownloaderAwarePriorityQueue.pqfactory>`
methods.
* The new ``key`` parameter displaced the ``startprios``
parameter 1 position to the right.
* The following class attributes have been added:
* :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.crawler`
* :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.downstream_queue_cls`
(details above)
* :attr:`~scrapy.core.scheduler.ScrapyPriorityQueue.key` (details above)
* The ``serialize`` attribute has been removed (details above)
The following changes affect specifically the
:class:`~scrapy.core.scheduler.ScrapyPriorityQueue` class and may affect
subclasses:
* A new :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.priority`
method has been added which, given a request, returns
``request.priority * -1``.
It is used in :meth:`~scrapy.core.scheduler.ScrapyPriorityQueue.push`
to make up for the removal of its ``priority`` parameter.
* The ``spider`` attribute has been removed. Use
:attr:`crawler.spider <scrapy.core.scheduler.ScrapyPriorityQueue.crawler>`
instead.
The following changes affect specifically the
:class:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue` class and may
affect subclasses:
* A new :attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.pqueues`
attribute offers a mapping of downloader slot names to the
corresponding instances of
:attr:`~scrapy.core.scheduler.DownloaderAwarePriorityQueue.downstream_queue_cls`.
(:issue:`3884`)
.. _release-1.8.0:
@ -26,7 +828,7 @@ Backward-incompatible changes
* Python 3.4 is no longer supported, and some of the minimum requirements of
Scrapy have also changed:
* cssselect_ 0.9.1
* :doc:`cssselect <cssselect:index>` 0.9.1
* cryptography_ 2.0
* lxml_ 3.5.0
* pyOpenSSL_ 16.2.0
@ -288,6 +1090,13 @@ Backward-incompatible changes
:class:`~scrapy.http.Request` objects instead of arbitrary Python data
structures.
* An additional ``crawler`` parameter has been added to the ``__init__``
method of the :class:`~scrapy.core.scheduler.Scheduler` class. Custom
scheduler subclasses which don't accept arbitrary parameters in their
``__init__`` method might break because of this change.
For more information, see :setting:`SCHEDULER`.
See also :ref:`1.7-deprecation-removals` below.
@ -1069,7 +1878,7 @@ Cleanups & Refactoring
~~~~~~~~~~~~~~~~~~~~~~
- Tests: remove temp files and folders (:issue:`2570`),
fixed ProjectUtilsTest on OS X (:issue:`2569`),
fixed ProjectUtilsTest on macOS (:issue:`2569`),
use portable pypy for Linux on Travis CI (:issue:`2710`)
- Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`)
- Remove “Python 3 progress” badge (:issue:`2567`)
@ -1609,7 +2418,7 @@ Deprecations and Removals
+ ``scrapy.utils.datatypes.SiteNode``
- The previously bundled ``scrapy.xlib.pydispatch`` library was deprecated and
replaced by `pydispatcher <https://pypi.python.org/pypi/PyDispatcher>`_.
replaced by `pydispatcher <https://pypi.org/project/PyDispatcher/>`_.
Relocations
@ -1638,7 +2447,7 @@ Bugfixes
- Makes ``_monkeypatches`` more robust (:issue:`1634`).
- Fixed bug on ``XMLItemExporter`` with non-string fields in
items (:issue:`1738`).
- Fixed startproject command in OS X (:issue:`1635`).
- Fixed startproject command in macOS (:issue:`1635`).
- Fixed :class:`~scrapy.exporters.PythonItemExporter` and CSVExporter for
non-string item types (:issue:`1737`).
- Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
@ -1706,12 +2515,12 @@ Scrapy 1.0.4 (2015-12-30)
- Typos corrections (:commit:`7067117`)
- fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`)
- Add note to Ubuntu install section about Debian compatibility (:commit:`23fda69`)
- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`)
- Replace alternative macOS install workaround with virtualenv (:commit:`98b63ee`)
- Reference Homebrew's homepage for installation instructions (:commit:`1925db1`)
- Add oldest supported tox version to contributing docs (:commit:`5d10d6d`)
- Note in install docs about pip being already included in python>=2.7.9 (:commit:`85c980e`)
- Add non-python dependencies to Ubuntu install section in the docs (:commit:`fbd010d`)
- Add OS X installation section to docs (:commit:`d8f4cba`)
- Add macOS installation section to docs (:commit:`d8f4cba`)
- DOC(ENH): specify path to rtd theme explicitly (:commit:`de73b1a`)
- minor: scrapy.Spider docs grammar (:commit:`1ddcc7b`)
- Make common practices sample code match the comments (:commit:`1b85bcf`)
@ -2443,7 +3252,7 @@ Other
~~~~~
- Dropped Python 2.6 support (:issue:`448`)
- Add `cssselect`_ python package as install dependency
- Add :doc:`cssselect <cssselect:index>` python package as install dependency
- Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on.
- Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support.
- Running test suite now requires ``mock`` python library (:issue:`390`)
@ -2564,7 +3373,7 @@ Scrapy 0.18.0 (released 2013-08-09)
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
- added from_crawler method to spiders
- added system tests with mock server
- more improvements to Mac OS compatibility (thanks Alex Cepoi)
- more improvements to macOS compatibility (thanks Alex Cepoi)
- several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez)
- support custom download slots
- added --spider option to "shell" command.
@ -2640,7 +3449,7 @@ Scrapy 0.16.3 (released 2012-12-07)
- Remove concurrency limitation when using download delays and still ensure inter-request delays are enforced (:commit:`487b9b5`)
- add error details when image pipeline fails (:commit:`8232569`)
- improve mac os compatibility (:commit:`8dcf8aa`)
- improve macOS compatibility (:commit:`8dcf8aa`)
- setup.py: use README.rst to populate long_description (:commit:`7b5310d`)
- doc: removed obsolete references to ClientForm (:commit:`80f9bb6`)
- correct docs for default storage backend (:commit:`2aa491b`)
@ -3040,17 +3849,16 @@ Scrapy 0.7
First release of Scrapy.
.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
.. _AJAX crawleable urls: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started?csw=1
.. _botocore: https://github.com/boto/botocore
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
.. _cryptography: https://cryptography.io/en/latest/
.. _cssselect: https://github.com/scrapy/cssselect/
.. _docstrings: https://docs.python.org/glossary.html#term-docstring
.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
.. _docstrings: https://docs.python.org/3/glossary.html#term-docstring
.. _KeyboardInterrupt: https://docs.python.org/3/library/exceptions.html#KeyboardInterrupt
.. _LevelDB: https://github.com/google/leveldb
.. _lxml: http://lxml.de/
.. _lxml: https://lxml.de/
.. _marshal: https://docs.python.org/2/library/marshal.html
.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
@ -3061,11 +3869,11 @@ First release of Scrapy.
.. _queuelib: https://github.com/scrapy/queuelib
.. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
.. _resource: https://docs.python.org/2/library/resource.html
.. _robots.txt: http://www.robotstxt.org/
.. _robots.txt: https://www.robotstxt.org/
.. _scrapely: https://github.com/scrapy/scrapely
.. _service_identity: https://service-identity.readthedocs.io/en/stable/
.. _six: https://six.readthedocs.io/
.. _tox: https://pypi.python.org/pypi/tox
.. _tox: https://pypi.org/project/tox/
.. _Twisted: https://twistedmatrix.com/trac/
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
.. _w3lib: https://github.com/scrapy/w3lib

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=2.1
sphinx-hoverxref
sphinx-notfound-page
sphinx_rtd_theme
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx_rtd_theme>=0.4

2
docs/topics/api.rst
View File

@ -91,7 +91,7 @@ how you :ref:`configure the downloader middlewares
provided while constructing the crawler, and it is created after the
arguments given in the :meth:`crawl` method.
.. method:: crawl(\*args, \**kwargs)
.. method:: crawl(*args, **kwargs)
Starts the crawler by instantiating its spider class with the given
``args`` and ``kwargs`` arguments, while setting the execution engine in

2
docs/topics/architecture.rst
View File

@ -104,7 +104,7 @@ Spiders
-------
Spiders are custom classes written by Scrapy users to parse responses and
extract items (aka scraped items) from them or additional requests to
extract :ref:`items <topics-items>` from them or additional requests to
follow. For more information see :ref:`topics-spiders`.
.. _component-pipelines:

28
docs/topics/asyncio.rst Normal file
View File

@ -0,0 +1,28 @@
=======
asyncio
=======
.. versionadded:: 2.0
Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
reactor <install-asyncio>`, you may use :mod:`asyncio` and
:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
versions may introduce related changes without a deprecation
period or warning.
.. _install-asyncio:
Installing the asyncio reactor
==============================
To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to
``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``.
If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to
install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`::
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')

9
docs/topics/broad-crawls.rst
View File

@ -188,7 +188,7 @@ AjaxCrawlMiddleware helps to crawl them correctly.
It is turned OFF by default because it has some performance overhead,
and enabling it for focused crawls doesn't make much sense.
.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started
.. _broad-crawls-bfo:
@ -211,3 +211,10 @@ If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
<broad-crawls-concurrency>` you should :ref:`debug your memory leaks
<topics-leaks>`.
Install a specific Twisted reactor
==================================
If the crawl is exceeding the system's capabilities, you might want to try
installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.

6
docs/topics/contracts.rst
View File

@ -78,7 +78,7 @@ override three methods:
.. module:: scrapy.contracts
.. class:: Contract(method, \*args)
.. class:: Contract(method, *args)
:param method: callback function to which the contract is associated
:type method: function
@ -136,7 +136,7 @@ Detecting check runs
====================
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
set to the ``true`` string. You can use `os.environ`_ to perform any change to
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
your spiders or your settings when ``scrapy check`` is used::
import os
@ -148,5 +148,3 @@ your spiders or your settings when ``scrapy check`` is used::
def __init__(self):
if os.environ.get('SCRAPY_CHECK'):
pass # Do some scraper adjustments when a check is running
.. _os.environ: https://docs.python.org/3/library/os.html#os.environ

112
docs/topics/coroutines.rst Normal file
View File

@ -0,0 +1,112 @@
==========
Coroutines
==========
.. versionadded:: 2.0
Scrapy has :ref:`partial support <coroutine-support>` for the
:ref:`coroutine syntax <async>`.
.. _coroutine-support:
Supported callables
===================
The following callables may be defined as coroutines using ``async def``, and
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.http.Request` callbacks.
The following are known caveats of the current implementation that we aim
to address in future versions of Scrapy:
- The callback output is not processed until the whole callback finishes.
As a side effect, if the callback raises an exception, none of its
output is processed.
- Because `asynchronous generators were introduced in Python 3.6`_, you
can only use ``yield`` if you are using Python 3.6 or later.
If you need to output multiple items or requests and you are using
Python 3.5, return an iterable (e.g. a list) instead.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
- The
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`,
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`,
and
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
methods of
:ref:`downloader middlewares <topics-downloader-middleware-custom>`.
- :ref:`Signal handlers that support deferreds <signal-deferred>`.
.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
Usage
=====
There are several use cases for coroutines in Scrapy. Code that would
return Deferreds when written for previous Scrapy versions, such as downloader
middlewares and signal handlers, can be rewritten to be shorter and cleaner::
from itemadapter import ItemAdapter
class DbPipeline:
def _update_item(self, data, item):
adapter = ItemAdapter(item)
adapter['field'] = data
return item
def process_item(self, item, spider):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter['id'])
dfd.addCallback(self._update_item, item)
return dfd
becomes::
from itemadapter import ItemAdapter
class DbPipeline:
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
adapter['field'] = await db.get_some_data(adapter['id'])
return item
Coroutines may be used to call asynchronous code. This includes other
coroutines, functions that return Deferreds and functions that return
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
class MySpider(Spider):
# ...
async def parse_with_deferred(self, response):
additional_response = await treq.get('https://additional.url')
additional_data = await treq.content(additional_response)
# ... use response and additional_data to yield items and requests
async def parse_with_asyncio(self, response):
async with aiohttp.ClientSession() as session:
async with session.get('https://additional.url') as additional_response:
additional_data = await r.text()
# ... use response and additional_data to yield items and requests
.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
:mod:`asyncio` loop and to use them you need to
:doc:`enable asyncio support in Scrapy<asyncio>`.
Common use cases for asynchronous code include:
* requesting data from websites, databases and other services (in callbacks,
pipelines and middlewares);
* storing data in databases (in pipelines and middlewares);
* delaying the spider initialization until some external event (in the
:signal:`spider_opened` handler);
* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
.. _aio-libs: https://github.com/aio-libs

5
docs/topics/developer-tools.rst
View File

@ -142,7 +142,7 @@ a use case:
Say you want to find the ``Next`` button on the page. Type ``Next`` into the
search bar on the top right of the `Inspector`. You should get two results.
The first is a ``li`` tag with the ``class="text"``, the second the text
The first is a ``li`` tag with the ``class="next"``, the second the text
of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
If you hover over the tag, you'll see the button highlighted. From here
we could easily create a :ref:`Link Extractor <topics-link-extractors>` to
@ -292,6 +292,9 @@ Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments.
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite

36
docs/topics/downloader-middleware.rst
View File

@ -202,6 +202,11 @@ CookiesMiddleware
sends them back on subsequent requests (from that spider), just like web
browsers do.
.. caution:: When non-UTF8 encoded byte sequences are passed to a
:class:`~scrapy.http.Request`, the ``CookiesMiddleware`` will log
a warning. Refer to :ref:`topics-logging-advanced-customization`
to customize the logging behaviour.
The following settings can be used to configure the cookie middleware:
* :setting:`COOKIES_ENABLED`
@ -259,8 +264,8 @@ COOKIES_DEBUG
Default: ``False``
If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie``
header) and all cookies received in responses (ie. ``Set-Cookie`` header).
If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie``
header) and all cookies received in responses (i.e. ``Set-Cookie`` header).
Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
@ -709,7 +714,7 @@ HttpCompressionMiddleware
provided `brotlipy`_ is installed.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.python.org/pypi/brotlipy
.. _brotlipy: https://pypi.org/project/brotlipy/
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -739,7 +744,7 @@ HttpProxyMiddleware
This middleware sets the HTTP proxy to use for requests, by setting the
``proxy`` meta value for :class:`~scrapy.http.Request` objects.
Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys
Like the Python standard library module :mod:`urllib.request`, it obeys
the following environment variables:
* ``http_proxy``
@ -751,9 +756,6 @@ HttpProxyMiddleware
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
environment variables, and it will also ignore ``no_proxy`` environment variable.
.. _urllib: https://docs.python.org/2/library/urllib.html
.. _urllib2: https://docs.python.org/2/library/urllib2.html
RedirectMiddleware
------------------
@ -829,6 +831,7 @@ REDIRECT_MAX_TIMES
Default: ``20``
The maximum number of redirections that will be followed for a single request.
After this maximum, the request's response is returned as is.
MetaRefreshMiddleware
---------------------
@ -868,10 +871,14 @@ Whether the Meta Refresh middleware will be enabled.
METAREFRESH_IGNORE_TAGS
^^^^^^^^^^^^^^^^^^^^^^^
Default: ``['script', 'noscript']``
Default: ``[]``
Meta tags within these tags are ignored.
.. versionchanged:: 2.0
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``['script', 'noscript']`` to ``[]``.
.. setting:: METAREFRESH_MAXDELAY
METAREFRESH_MAXDELAY
@ -1032,13 +1039,12 @@ Scrapy uses this parser by default.
RobotFileParser
~~~~~~~~~~~~~~~
Based on `RobotFileParser
<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
Based on :class:`~urllib.robotparser.RobotFileParser`:
* is Python's built-in robots.txt_ parser
* is compliant with `Martijn Koster's 1996 draft specification
<http://www.robotstxt.org/norobots-rfc.txt>`_
<https://www.robotstxt.org/norobots-rfc.txt>`_
* lacks support for wildcard matching
@ -1061,7 +1067,7 @@ Based on `Reppy <https://github.com/seomoz/reppy/>`_:
<https://github.com/seomoz/rep-cpp>`_
* is compliant with `Martijn Koster's 1996 draft specification
<http://www.robotstxt.org/norobots-rfc.txt>`_
<https://www.robotstxt.org/norobots-rfc.txt>`_
* supports wildcard matching
@ -1086,7 +1092,7 @@ Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
* implemented in Python
* is compliant with `Martijn Koster's 1996 draft specification
<http://www.robotstxt.org/norobots-rfc.txt>`_
<https://www.robotstxt.org/norobots-rfc.txt>`_
* supports wildcard matching
@ -1115,7 +1121,7 @@ implementing the methods described below.
.. autoclass:: RobotParser
:members:
.. _robots.txt: http://www.robotstxt.org/
.. _robots.txt: https://www.robotstxt.org/
DownloaderStats
---------------
@ -1155,7 +1161,7 @@ AjaxCrawlMiddleware
Middleware that finds 'AJAX crawlable' page variants based
on meta-fragment html tag. See
https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
https://developers.google.com/search/docs/ajax-crawling/docs/getting-started
for more info.
.. note::

32
docs/topics/dynamic-content.rst
View File

@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is
probably not your request, but the target server. The target server might be
buggy, overloaded, or :ref:`banning <bans>` some of your requests.
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
.. _topics-handling-response-formats:
Handling different response formats
@ -115,7 +118,7 @@ data from it depends on the type of response:
- If the response is HTML or XML, use :ref:`selectors
<topics-selectors>` as usual.
- If the response is JSON, use `json.loads`_ to load the desired data from
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`::
data = json.loads(response.text)
@ -130,8 +133,9 @@ data from it depends on the type of response:
- If the response is JavaScript, or HTML with a ``<script/>`` element
containing the desired data, see :ref:`topics-parsing-javascript`.
- If the response is CSS, use a `regular expression`_ to extract the desired
data from :attr:`response.text <scrapy.http.TextResponse.text>`.
- If the response is CSS, use a :doc:`regular expression <library/re>` to
extract the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`.
.. _topics-parsing-images:
@ -168,8 +172,9 @@ JavaScript code:
Once you have a string with the JavaScript code, you can extract the desired
data from it:
- You might be able to use a `regular expression`_ to extract the desired
data in JSON format, which you can then parse with `json.loads`_.
- You might be able to use a :doc:`regular expression <library/re>` to
extract the desired data in JSON format, which you can then parse with
:func:`json.loads`.
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows:
@ -179,6 +184,18 @@ data from it:
>>> json.loads(json_data)
{'field': 'value'}
- chompjs_ provides an API to parse JavaScript objects into a :class:`dict`.
For example, if the JavaScript code contains
``var data = {field: "value", secondField: "second value"};``
you can extract that data as follows:
>>> import chompjs
>>> javascript = response.css('script::text').get()
>>> data = chompjs.parse_js_object(javascript)
>>> data
{'field': 'value', 'secondField': 'second value'}
- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
that you can parse using :ref:`selectors <topics-selectors>`.
@ -236,17 +253,16 @@ along with `scrapy-selenium`_ for seamless integration.
.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
.. _chompjs: https://github.com/Nykakin/chompjs
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
.. _curl: https://curl.haxx.se/
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _js2xml: https://github.com/scrapinghub/js2xml
.. _json.loads: https://docs.python.org/library/json.html#json.loads
.. _pytesseract: https://github.com/madmaze/pytesseract
.. _regular expression: https://docs.python.org/library/re.html
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
.. _Selenium: https://www.seleniumhq.org/
.. _Selenium: https://www.selenium.dev/
.. _Splash: https://github.com/scrapinghub/splash
.. _tabula-py: https://github.com/chezou/tabula-py
.. _wget: https://www.gnu.org/software/wget/

4
docs/topics/email.rst
View File

@ -7,7 +7,7 @@ Sending e-mail
.. module:: scrapy.mail
:synopsis: Email sending facility
Although Python makes sending e-mails relatively easy via the `smtplib`_
Although Python makes sending e-mails relatively easy via the :mod:`smtplib`
library, Scrapy provides its own facility for sending e-mails which is very
easy to use and it's implemented using :doc:`Twisted non-blocking IO
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
@ -15,8 +15,6 @@ IO of the crawler. It also provides a simple API for sending attachments and
it's very easy to configure, with a few :ref:`settings
<topics-email-settings>`.
.. _smtplib: https://docs.python.org/2/library/smtplib.html
Quick example
=============

49
docs/topics/exceptions.rst
View File

@ -14,13 +14,6 @@ Built-in Exceptions reference
Here's a list of all exceptions included in Scrapy and their usage.
DropItem
--------
.. exception:: DropItem
The exception that must be raised by item pipeline stages to stop processing an
Item. For more information see :ref:`topics-item-pipeline`.
CloseSpider
-----------
@ -47,6 +40,14 @@ DontCloseSpider
This exception can be raised in a :signal:`spider_idle` signal handler to
prevent the spider from being closed.
DropItem
--------
.. exception:: DropItem
The exception that must be raised by item pipeline stages to stop processing an
Item. For more information see :ref:`topics-item-pipeline`.
IgnoreRequest
-------------
@ -77,3 +78,37 @@ NotSupported
This exception is raised to indicate an unsupported feature.
StopDownload
-------------
.. versionadded:: 2.2
.. exception:: StopDownload(fail=True)
Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
indicate that no further bytes should be downloaded for a response.
The ``fail`` boolean parameter controls which method will handle the resulting
response:
* If ``fail=True`` (default), the request errback is called. The response object is
available as the ``response`` attribute of the ``StopDownload`` exception,
which is in turn stored as the ``value`` attribute of the received
:class:`~twisted.python.failure.Failure` object. This means that in an errback
defined as ``def errback(self, failure)``, the response can be accessed though
``failure.value.response``.
* If ``fail=False``, the request callback is called instead.
In both cases, the response could have its body truncated: the body contains
all bytes received up until the exception is raised, including the bytes
received in the signal handler that raises the exception. Also, the response
object is marked with ``"download_stopped"`` in its :attr:`Response.flags`
attribute.
.. note:: ``fail`` is a keyword-only parameter, i.e. raising
``StopDownload(False)`` or ``StopDownload(True)`` will raise
a :class:`TypeError`.
See the documentation for the :class:`~scrapy.signals.bytes_received` signal
and the :ref:`topics-stop-response-download` topic for additional information and examples.

73
docs/topics/exporters.rst
View File

@ -40,9 +40,10 @@ Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
from itemadapter import ItemAdapter
from scrapy.exporters import XmlItemExporter
class PerYearXmlExportPipeline(object):
class PerYearXmlExportPipeline:
"""Distribute items across multiple XML files according to their 'year' field"""
def open_spider(self, spider):
@ -53,7 +54,8 @@ value of one of their fields::
exporter.finish_exporting()
def _exporter_for_item(self, item):
year = item['year']
adapter = ItemAdapter(item)
year = adapter['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
exporter = XmlItemExporter(f)
@ -137,7 +139,7 @@ output examples, which assume you're exporting these two items::
BaseItemExporter
----------------
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
This is the (abstract) base class for all Item Exporters. It provides
support for common features used by all (concrete) Item Exporters, such as
@ -148,6 +150,9 @@ BaseItemExporter
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
.. versionadded:: 2.0
The *dont_fail* parameter.
.. method:: export_item(item)
Exports the given item. This method must be implemented in subclasses.
@ -164,9 +169,10 @@ BaseItemExporter
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
:param field: the field being serialized. If a raw dict is being
exported (not :class:`~.Item`) *field* value is an empty dict.
:type field: :class:`~scrapy.item.Field` object or an empty dict
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
:class:`dict`.
:type field: :class:`~scrapy.item.Field` object or a :class:`dict` instance
:param name: the name of the field being serialized
:type name: str
@ -189,14 +195,17 @@ BaseItemExporter
.. attribute:: fields_to_export
A list with the name of the fields that will be exported, or None if you
want to export all fields. Defaults to None.
A list with the name of the fields that will be exported, or ``None`` if
you want to export all fields. Defaults to ``None``.
Some exporters (like :class:`CsvItemExporter`) respect the order of the
fields defined in this attribute.
Some exporters may require fields_to_export list in order to export the
data properly when spiders return dicts (not :class:`~Item` instances).
When using :ref:`item objects <item-types>` that do not expose all their
possible fields, exporters that do not support exporting a different
subset of fields per item will only export the fields found in the first
item exported. Use ``fields_to_export`` to define all the fields to be
exported.
.. attribute:: export_empty_fields
@ -233,9 +242,9 @@ PythonItemExporter
XmlItemExporter
---------------
.. class:: XmlItemExporter(file, item_element='item', root_element='items', \**kwargs)
.. class:: XmlItemExporter(file, item_element='item', root_element='items', **kwargs)
Exports Items in XML format to the specified file object.
Exports items in XML format to the specified file object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -287,9 +296,9 @@ XmlItemExporter
CsvItemExporter
---------------
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', \**kwargs)
.. class:: CsvItemExporter(file, include_headers_line=True, join_multivalued=',', **kwargs)
Exports Items in CSV format to the given file-like object. If the
Exports items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
CSV columns and their order. The :attr:`export_empty_fields` attribute has
no effect on this exporter.
@ -308,7 +317,7 @@ CsvItemExporter
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
:func:`csv.writer` function, so you can use any :func:`csv.writer` function
argument to customize this exporter.
A typical output of this exporter would be::
@ -317,14 +326,12 @@ CsvItemExporter
Color TV,1200
DVD player,200
.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer
PickleItemExporter
------------------
.. class:: PickleItemExporter(file, protocol=0, \**kwargs)
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
Exports Items in pickle format to the given file-like object.
Exports items in pickle format to the given file-like object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -332,21 +339,19 @@ PickleItemExporter
:param protocol: The pickle protocol to use.
:type protocol: int
For more information, refer to the `pickle module documentation`_.
For more information, see :mod:`pickle`.
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
Pickle isn't a human readable format, so no output examples are provided.
.. _pickle module documentation: https://docs.python.org/2/library/pickle.html
PprintItemExporter
------------------
.. class:: PprintItemExporter(file, \**kwargs)
.. class:: PprintItemExporter(file, **kwargs)
Exports Items in pretty print format to the specified file object.
Exports items in pretty print format to the specified file object.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -364,13 +369,13 @@ PprintItemExporter
JsonItemExporter
----------------
.. class:: JsonItemExporter(file, \**kwargs)
.. class:: JsonItemExporter(file, **kwargs)
Exports Items in JSON format to the specified file-like object, writing all
Exports items in JSON format to the specified file-like object, writing all
objects as a list of objects. The additional ``__init__`` method arguments are
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -390,18 +395,16 @@ JsonItemExporter
stream-friendly format, consider using :class:`JsonLinesItemExporter`
instead, or splitting the output in multiple chunks.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
JsonLinesItemExporter
---------------------
.. class:: JsonLinesItemExporter(file, \**kwargs)
.. class:: JsonLinesItemExporter(file, **kwargs)
Exports Items in JSON format to the specified file-like object, writing one
Exports items in JSON format to the specified file-like object, writing one
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
``__init__`` method argument to customize this exporter.
the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -414,8 +417,6 @@ JsonLinesItemExporter
Unlike the one produced by :class:`JsonItemExporter`, the format produced by
this exporter is well suited for serializing large amounts of data.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
MarshalItemExporter
-------------------

11
docs/topics/extensions.rst
View File

@ -63,7 +63,7 @@ but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set.
Disabling an extension
======================
In order to disable an extension that comes enabled by default (ie. those
In order to disable an extension that comes enabled by default (i.e. those
included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
``None``. For example::
@ -107,7 +107,7 @@ Here is the code of such extension::
logger = logging.getLogger(__name__)
class SpiderOpenCloseLogging(object):
class SpiderOpenCloseLogging:
def __init__(self, item_count):
self.item_count = item_count
@ -345,7 +345,7 @@ signal is received. The information dumped is the following:
After the stack trace and engine status is dumped, the Scrapy process continues
running normally.
This extension only works on POSIX-compliant platforms (ie. not Windows),
This extension only works on POSIX-compliant platforms (i.e. not Windows),
because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows.
There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
@ -364,13 +364,12 @@ Debugger extension
.. class:: Debugger
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
Invokes a :doc:`Python debugger <library/pdb>` inside a running Scrapy process when a `SIGUSR2`_
signal is received. After the debugger is exited, the Scrapy process continues
running normally.
For more info see `Debugging in Python`_.
This extension only works on POSIX-compliant platforms (ie. not Windows).
This extension only works on POSIX-compliant platforms (i.e. not Windows).
.. _Python debugger: https://docs.python.org/2/library/pdb.html
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/

90
docs/topics/feed-exports.rst
View File

@ -12,7 +12,7 @@ generating an "export file" with the scraped data (commonly called "export
feed") to be consumed by other systems.
Scrapy provides this functionality out of the box with the Feed Exports, which
allows you to generate a feed with the scraped items, using multiple
allows you to generate feeds with the scraped items, using multiple
serialization formats and storage backends.
.. _topics-feed-format:
@ -36,7 +36,7 @@ But you can also extend the supported format through the
JSON
----
* :setting:`FEED_FORMAT`: ``json``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
* Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
* See :ref:`this warning <json-with-large-data>` if you're using JSON with
large feeds.
@ -46,7 +46,7 @@ JSON
JSON lines
----------
* :setting:`FEED_FORMAT`: ``jsonlines``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
* Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
.. _topics-feed-format-csv:
@ -54,7 +54,7 @@ JSON lines
CSV
---
* :setting:`FEED_FORMAT`: ``csv``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
* To specify columns to export and their order use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
@ -66,7 +66,7 @@ CSV
XML
---
* :setting:`FEED_FORMAT`: ``xml``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
* Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
.. _topics-feed-format-pickle:
@ -74,7 +74,7 @@ XML
Pickle
------
* :setting:`FEED_FORMAT`: ``pickle``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
* Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
.. _topics-feed-format-marshal:
@ -82,7 +82,7 @@ Pickle
Marshal
-------
* :setting:`FEED_FORMAT`: ``marshal``
* Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
* Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
@ -91,8 +91,8 @@ Marshal
Storages
========
When using the feed exports you define where to store the feed using a URI_
(through the :setting:`FEED_URI` setting). The feed exports supports multiple
When using the feed exports you define where to store the feed using one or multiple URIs_
(through the :setting:`FEEDS` setting). The feed exports supports multiple
storage backend types which are defined by the URI scheme.
The storages backends supported out of the box are:
@ -211,38 +211,66 @@ Settings
These are the settings used for configuring the feed exports:
* :setting:`FEED_URI` (mandatory)
* :setting:`FEED_FORMAT`
* :setting:`FEEDS` (mandatory)
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
* :setting:`FEED_STORAGES`
* :setting:`FEED_STORAGE_FTP_ACTIVE`
* :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
.. currentmodule:: scrapy.extensions.feedexport
.. setting:: FEED_URI
.. setting:: FEEDS
FEED_URI
--------
FEEDS
-----
Default: ``None``
.. versionadded:: 2.1
The URI of the export feed. See :ref:`topics-feed-storage-backends` for
supported URI schemes.
Default: ``{}``
This setting is required for enabling the feed exports.
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
object) and each value is a nested dictionary containing configuration
parameters for the specific feed.
This setting is required for enabling the feed export feature.
.. setting:: FEED_FORMAT
See :ref:`topics-feed-storage-backends` for supported URI schemes.
FEED_FORMAT
-----------
For instance::
The serialization format to be used for the feed. See
:ref:`topics-feed-format` for possible values.
{
'items.json': {
'format': 'json',
'encoding': 'utf8',
'store_empty': False,
'fields': None,
'indent': 4,
},
'/home/user/documents/items.xml': {
'format': 'xml',
'fields': ['name', 'price'],
'encoding': 'latin1',
'indent': 8,
},
pathlib.Path('items.csv'): {
'format': 'csv',
'fields': ['price', 'name'],
},
}
The following is a list of the accepted keys and the setting that is used
as a fallback value if that key is not provided for a specific feed definition.
* ``format``: the serialization format to be used for the feed.
See :ref:`topics-feed-format` for possible values.
Mandatory, no fallback setting
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
.. setting:: FEED_EXPORT_ENCODING
@ -270,8 +298,8 @@ Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
Use FEED_EXPORT_FIELDS option to define fields to export and their order.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
defined in dicts or :class:`~.Item` subclasses a spider is yielding.
When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
defined in :ref:`item objects <topics-items>` yielded by your spider.
If an exporter requires a fixed set of fields (this is the case for
:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
@ -301,7 +329,7 @@ FEED_STORE_EMPTY
Default: ``False``
Whether to export empty feeds (ie. feeds with no items).
Whether to export empty feeds (i.e. feeds with no items).
.. setting:: FEED_STORAGES
@ -397,7 +425,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
'csv': None,
}
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl

89
docs/topics/item-pipeline.rst
View File

@ -27,15 +27,19 @@ Each item pipeline component is a Python class that must implement the following
.. method:: process_item(self, item, spider)
This method is called for every item pipeline component. :meth:`process_item`
must either: return a dict with data, return an :class:`~scrapy.item.Item`
(or any descendant class) object, return a
:class:`~twisted.internet.defer.Deferred` or raise
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
processed by further pipeline components.
This method is called for every item pipeline component.
:param item: the item scraped
:type item: :class:`~scrapy.item.Item` object or a dict
`item` is an :ref:`item object <item-types>`, see
:ref:`supporting-item-types`.
:meth:`process_item` must either: return an :ref:`item object <item-types>`,
return a :class:`~twisted.internet.defer.Deferred` or raise a
:exc:`~scrapy.exceptions.DropItem` exception.
Dropped items are no longer processed by further pipeline components.
:param item: the scraped item
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
@ -79,16 +83,17 @@ Let's take a look at the following hypothetical pipeline that adjusts the
(``price_excludes_vat`` attribute), and drops those items which don't
contain a price::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class PricePipeline(object):
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
if item.get('price'):
if item.get('price_excludes_vat'):
item['price'] = item['price'] * self.vat_factor
adapter = ItemAdapter(item)
if adapter.get('price'):
if adapter.get('price_excludes_vat'):
adapter['price'] = adapter['price'] * self.vat_factor
return item
else:
raise DropItem("Missing price in %s" % item)
@ -103,7 +108,9 @@ format::
import json
class JsonWriterPipeline(object):
from itemadapter import ItemAdapter
class JsonWriterPipeline:
def open_spider(self, spider):
self.file = open('items.jl', 'w')
@ -112,7 +119,7 @@ format::
self.file.close()
def process_item(self, item, spider):
line = json.dumps(dict(item)) + "\n"
line = json.dumps(ItemAdapter(item).asdict()) + "\n"
self.file.write(line)
return item
@ -131,8 +138,9 @@ The main point of this example is to show how to use :meth:`from_crawler`
method and how to clean up the resources properly.::
import pymongo
from itemadapter import ItemAdapter
class MongoPipeline(object):
class MongoPipeline:
collection_name = 'scrapy_items'
@ -155,57 +163,60 @@ method and how to clean up the resources properly.::
self.client.close()
def process_item(self, item, spider):
self.db[self.collection_name].insert_one(dict(item))
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
return item
.. _MongoDB: https://www.mongodb.org/
.. _pymongo: https://api.mongodb.org/python/current/
.. _MongoDB: https://www.mongodb.com/
.. _pymongo: https://api.mongodb.com/python/current/
.. _ScreenshotPipeline:
Take screenshot of item
-----------------------
This example demonstrates how to return a
:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
It uses Splash_ to render screenshot of item url. Pipeline
makes request to locally running instance of Splash_. After request is downloaded
and Deferred callback fires, it saves item to a file and adds filename to an item.
This example demonstrates how to use :doc:`coroutine syntax <coroutines>` in
the :meth:`process_item` method.
This item pipeline makes a request to a locally-running instance of Splash_ to
render a screenshot of the item URL. After the request response is downloaded,
the item pipeline saves the screenshot to a file and adds the filename to the
item.
::
import scrapy
import hashlib
from urllib.parse import quote
import scrapy
from itemadapter import ItemAdapter
class ScreenshotPipeline(object):
class ScreenshotPipeline:
"""Pipeline that uses Splash to render screenshot of
every Scrapy item."""
SPLASH_URL = "http://localhost:8050/render.png?url={}"
def process_item(self, item, spider):
encoded_item_url = quote(item["url"])
async def process_item(self, item, spider):
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url)
dfd = spider.crawler.engine.download(request, spider)
dfd.addBoth(self.return_item, item)
return dfd
response = await spider.crawler.engine.download(request, spider)
def return_item(self, response, item):
if response.status != 200:
# Error happened, return item.
return item
# Save screenshot to file, filename will be hash of url.
url = item["url"]
url = adapter["url"]
url_hash = hashlib.md5(url.encode("utf8")).hexdigest()
filename = "{}.png".format(url_hash)
with open(filename, "wb") as f:
f.write(response.body)
# Store filename in item.
item["screenshot_filename"] = filename
adapter["screenshot_filename"] = filename
return item
.. _Splash: https://splash.readthedocs.io/en/stable/
@ -218,18 +229,20 @@ already processed. Let's say that our items have a unique id, but our spider
returns multiples items with the same id::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
class DuplicatesPipeline(object):
class DuplicatesPipeline:
def __init__(self):
self.ids_seen = set()
def process_item(self, item, spider):
if item['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %s" % item)
adapter = ItemAdapter(item)
if adapter['id'] in self.ids_seen:
raise DropItem("Duplicate item found: %r" % item)
else:
self.ids_seen.add(item['id'])
self.ids_seen.add(adapter['id'])
return item

264
docs/topics/items.rst
View File

@ -8,31 +8,155 @@ Items
:synopsis: Item and Field classes
The main goal in scraping is to extract structured data from unstructured
sources, typically, web pages. Scrapy spiders can return the extracted data
as Python dicts. While convenient and familiar, Python dicts lack structure:
it is easy to make a typo in a field name or return inconsistent data,
especially in a larger project with many spiders.
sources, typically, web pages. :ref:`Spiders <topics-spiders>` may return the
extracted data as `items`, Python objects that define key-value pairs.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide a `dictionary-like`_ API with a convenient syntax for declaring
their available fields.
Scrapy supports :ref:`multiple types of items <item-types>`. When you create an
item, you may use whichever type of item you want. When you write code that
receives an item, your code should :ref:`work for any item type
<supporting-item-types>`.
Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _item-types:
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
Item Types
==========
Scrapy supports the following types of items, via the `itemadapter`_ library:
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
.. _itemadapter: https://github.com/scrapy/itemadapter
.. _dict-items:
Dictionaries
------------
As an item type, :class:`dict` is convenient and familiar.
.. _item-objects:
Item objects
------------
:class:`Item` provides a :class:`dict`-like API plus additional features that
make it the most feature-complete item type:
.. class:: Item([arg])
:class:`Item` objects replicate the standard :class:`dict` API, including
its ``__init__`` method.
:class:`Item` allows defining field names, so that:
- :class:`KeyError` is raised when using undefined field names (i.e.
prevents typos going unnoticed)
- :ref:`Item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all
of them
:class:`Item` also allows defining field metadata, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
(see :ref:`topics-leaks-trackrefs`).
:class:`Item` objects also provide the following additional API members:
.. automethod:: copy
.. automethod:: deepcopy
.. attribute:: fields
A dictionary containing *all declared fields* for this Item, not only
those populated. The keys are the field names and the values are the
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
Example::
from scrapy.item import Item, Field
class CustomItem(Item):
one_field = Field()
another_field = Field()
.. _dataclass-items:
Dataclass objects
-----------------
.. versionadded:: 2.2
:func:`~dataclasses.dataclass` allows defining item classes with field names,
so that :ref:`item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all of them.
Additionally, ``dataclass`` items also allow to:
* define the type and default value of each defined field.
* define custom field metadata through :func:`dataclasses.field`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
They work natively in Python 3.7 or later, or using the `dataclasses
backport`_ in Python 3.6.
.. _dataclasses backport: https://pypi.org/project/dataclasses/
Example::
from dataclasses import dataclass
@dataclass
class CustomItem:
one_field: str
another_field: int
.. note:: Field types are not enforced at run time.
.. _attrs-items:
attr.s objects
--------------
.. versionadded:: 2.2
:func:`attr.s` allows defining item classes with field names,
so that :ref:`item exporters <topics-exporters>` can export all fields by
default even if the first scraped object does not have values for all of them.
Additionally, ``attr.s`` items also allow to:
* define the type and default value of each defined field.
* define custom field :ref:`metadata <attrs:metadata>`, which can be used to
:ref:`customize serialization <topics-exporters-field-serialization>`.
In order to use this type, the :doc:`attrs package <attrs:index>` needs to be installed.
Example::
import attr
@attr.s
class CustomItem:
one_field = attr.ib()
another_field = attr.ib()
Working with Item objects
=========================
.. _topics-items-declaring:
Declaring Items
===============
Declaring Item subclasses
-------------------------
Items are declared using a simple class definition syntax and :class:`Field`
objects. Here is an example::
Item subclasses are declared using a simple class definition syntax and
:class:`Field` objects. Here is an example::
import scrapy
@ -50,10 +174,11 @@ objects. Here is an example::
.. _Django: https://www.djangoproject.com/
.. _Django Models: https://docs.djangoproject.com/en/dev/topics/db/models/
.. _topics-items-fields:
Item Fields
===========
Declaring fields
----------------
:class:`Field` objects are used to specify metadata for each field. For
example, the serializer function for the ``last_updated`` field illustrated in
@ -74,15 +199,31 @@ It's important to note that the :class:`Field` objects used to declare the item
do not stay assigned as class attributes. Instead, they can be accessed through
the :attr:`Item.fields` attribute.
Working with Items
==================
.. class:: Field([arg])
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
doesn't provide any extra functionality or attributes. In other words,
:class:`Field` objects are plain-old Python dicts. A separate class is used
to support the :ref:`item declaration syntax <topics-items-declaring>`
based on class attributes.
.. note:: Field metadata can also be declared for ``dataclass`` and ``attrs``
items. Please refer to the documentation for `dataclasses.field`_ and
`attr.ib`_ for additional information.
.. _dataclasses.field: https://docs.python.org/3/library/dataclasses.html#dataclasses.field
.. _attr.ib: https://www.attrs.org/en/stable/api.html#attr.ib
Working with Item objects
-------------------------
Here are some examples of common tasks performed with items, using the
``Product`` item :ref:`declared above <topics-items-declaring>`. You will
notice the API is very similar to the `dict API`_.
notice the API is very similar to the :class:`dict` API.
Creating items
--------------
''''''''''''''
>>> product = Product(name='Desktop PC', price=1000)
>>> print(product)
@ -90,7 +231,7 @@ Product(name='Desktop PC', price=1000)
Getting field values
--------------------
''''''''''''''''''''
>>> product['name']
Desktop PC
@ -130,7 +271,7 @@ False
Setting field values
--------------------
''''''''''''''''''''
>>> product['last_updated'] = 'today'
>>> product['last_updated']
@ -143,9 +284,9 @@ KeyError: 'Product does not support field: lala'
Accessing all populated values
------------------------------
''''''''''''''''''''''''''''''
To access all populated values, just use the typical `dict API`_:
To access all populated values, just use the typical :class:`dict` API:
>>> product.keys()
['price', 'name']
@ -157,16 +298,14 @@ To access all populated values, just use the typical `dict API`_:
.. _copying-items:
Copying items
-------------
'''''''''''''
To copy an item, you must first decide whether you want a shallow copy or a
deep copy.
If your item contains mutable_ values like lists or dictionaries, a shallow
copy will keep references to the same mutable values across all different
copies.
.. _mutable: https://docs.python.org/glossary.html#term-mutable
If your item contains :term:`mutable` values like lists or dictionaries,
a shallow copy will keep references to the same mutable values across all
different copies.
For example, if you have an item with a list of tags, and you create a shallow
copy of that item, both the original item and the copy have the same list of
@ -175,9 +314,7 @@ other item as well.
If that is not the desired behavior, use a deep copy instead.
See the `documentation of the copy module`_ for more information.
.. _documentation of the copy module: https://docs.python.org/library/copy.html
See :mod:`copy` for more information.
To create a shallow copy of an item, you can either call
:meth:`~scrapy.item.Item.copy` on an existing item
@ -189,7 +326,7 @@ To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
Other common tasks
------------------
''''''''''''''''''
Creating dicts from items:
@ -207,8 +344,8 @@ Traceback (most recent call last):
KeyError: 'Product does not support field: lala'
Extending Items
===============
Extending Item subclasses
-------------------------
You can extend Items (to add more fields or to change some metadata for some
fields) by declaring a subclass of your original Item.
@ -228,46 +365,25 @@ appending more values, or changing existing values, like this::
That adds (or replaces) the ``serializer`` metadata key for the ``name`` field,
keeping all the previously existing metadata values.
Item objects
============
.. class:: Item([arg])
.. _supporting-item-types:
Return a new Item optionally initialized from the given argument.
Supporting All Item Types
=========================
Items replicate the standard `dict API`_, including its ``__init__`` method, and
also provide the following additional API members:
In code that receives an item, such as methods of :ref:`item pipelines
<topics-item-pipeline>` or :ref:`spider middlewares
<topics-spider-middleware>`, it is a good practice to use the
:class:`~itemadapter.ItemAdapter` class and the
:func:`~itemadapter.is_item` function to write code that works for
any :ref:`supported item type <item-types>`:
.. automethod:: copy
.. autoclass:: itemadapter.ItemAdapter
.. automethod:: deepcopy
.. attribute:: fields
A dictionary containing *all declared fields* for this Item, not only
those populated. The keys are the field names and the values are the
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict
Field objects
=============
.. class:: Field([arg])
The :class:`Field` class is just an alias to the built-in `dict`_ class and
doesn't provide any extra functionality or attributes. In other words,
:class:`Field` objects are plain-old Python dicts. A separate class is used
to support the :ref:`item declaration syntax <topics-items-declaring>`
based on class attributes.
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
.. autofunction:: itemadapter.is_item
Other classes related to Item
=============================
.. autoclass:: BaseItem
Other classes related to items
==============================
.. autoclass:: ItemMeta

5
docs/topics/jobs.rst
View File

@ -22,7 +22,7 @@ Job directory
To enable persistence support you just need to define a *job directory* through
the ``JOBDIR`` setting. This directory will be for storing all required data to
keep the state of a single job (ie. a spider run). It's important to note that
keep the state of a single job (i.e. a spider run). It's important to note that
this directory must not be shared by different spiders, or even different
jobs/runs of the same spider, as it's meant to be used for storing the state of
a *single* job.
@ -68,6 +68,9 @@ Cookies may expire. So, if you don't resume your spider quickly the requests
scheduled may no longer work. This won't be an issue if you spider doesn't rely
on cookies.
.. _request-serialization:
Request serialization
---------------------

91
docs/topics/leaks.rst
View File

@ -4,7 +4,7 @@
Debugging memory leaks
======================
In Scrapy, objects such as Requests, Responses and Items have a finite
In Scrapy, objects such as requests, responses and items have a finite
lifetime: they are created, used for a while, and finally destroyed.
From all those objects, the Request is probably the one with the longest
@ -17,8 +17,8 @@ what is known as a "memory leak".
To help debugging memory leaks, Scrapy provides a built-in mechanism for
tracking objects references called :ref:`trackref <topics-leaks-trackrefs>`,
and you can also use a third-party library called :ref:`Guppy
<topics-leaks-guppy>` for more advanced memory debugging (see below for more
and you can also use a third-party library called :ref:`muppy
<topics-leaks-muppy>` for more advanced memory debugging (see below for more
info). Both mechanisms must be used from the :ref:`Telnet Console
<topics-telnetconsole>`.
@ -61,8 +61,8 @@ Debugging memory leaks with ``trackref``
========================================
:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
memory leaks. It basically tracks the references to all live Requests,
Responses, Item and Selector objects.
memory leaks. It basically tracks the references to all live Request,
Response, Item, Spider and Selector objects.
You can enter the telnet console and inspect how many objects (of the classes
mentioned above) are currently alive using the ``prefs()`` function which is an
@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
.. class:: object_ref
Inherit from this class (instead of object) if you want to track live
Inherit from this class if you want to track live
instances with the ``trackref`` module.
.. function:: print_live_refs(class_name, ignore=NoneType)
@ -193,73 +193,18 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
``None`` if none is found. Use :func:`print_live_refs` first to get a list
of all tracked live objects per class name.
.. _topics-leaks-guppy:
Debugging memory leaks with Guppy
=================================
``trackref`` provides a very convenient mechanism for tracking down memory
leaks, but it only keeps track of the objects that are more likely to cause
memory leaks (Requests, Responses, Items, and Selectors). However, there are
other cases where the memory leaks could come from other (more or less obscure)
objects. If this is your case, and you can't find your leaks using ``trackref``,
you still have another resource: the `Guppy library`_.
If you're using Python3, see :ref:`topics-leaks-muppy`.
.. _Guppy library: https://pypi.python.org/pypi/guppy
If you use ``pip``, you can install Guppy with the following command::
pip install guppy
The telnet console also comes with a built-in shortcut (``hpy``) for accessing
Guppy heap objects. Here's an example to view all Python objects available in
the heap using Guppy:
>>> x = hpy.heap()
>>> x.bytype
Partition of a set of 297033 objects. Total size = 52587824 bytes.
Index Count % Size % Cumulative % Type
0 22307 8 16423880 31 16423880 31 dict
1 122285 41 12441544 24 28865424 55 str
2 68346 23 5966696 11 34832120 66 tuple
3 227 0 5836528 11 40668648 77 unicode
4 2461 1 2222272 4 42890920 82 type
5 16870 6 2024400 4 44915320 85 function
6 13949 5 1673880 3 46589200 89 types.CodeType
7 13422 5 1653104 3 48242304 92 list
8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
<1676 more rows. Type e.g. '_.more' to view.>
You can see that most space is used by dicts. Then, if you want to see from
which attribute those dicts are referenced, you could do:
>>> x.bytype[0].byvia
Partition of a set of 22307 objects. Total size = 16423880 bytes.
Index Count % Size % Cumulative % Referred Via:
0 10982 49 9416336 57 9416336 57 '.__dict__'
1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
2 3097 14 1122904 7 13220744 80
3 990 4 277200 2 13497944 82 "['cookies']"
4 987 4 276360 2 13774304 84 "['cache']"
5 985 4 275800 2 14050104 86 "['meta']"
6 897 4 251160 2 14301264 87 '[2]'
7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
8 672 3 188160 1 14686312 89 "['cb_kwargs']"
9 27 0 155016 1 14841328 90 '[1]'
<333 more rows. Type e.g. '_.more' to view.>
As you can see, the Guppy module is very powerful but also requires some deep
knowledge about Python internals. For more info about Guppy, refer to the
`Guppy documentation`_.
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
.. _topics-leaks-muppy:
Debugging memory leaks with muppy
=================================
``trackref`` provides a very convenient mechanism for tracking down memory
leaks, but it only keeps track of the objects that are more likely to cause
memory leaks. However, there are other cases where the memory leaks could come
from other (more or less obscure) objects. If this is your case, and you can't
find your leaks using ``trackref``, you still have another resource: the muppy
library.
You can use muppy from `Pympler`_.
.. _Pympler: https://pypi.org/project/Pympler/
@ -311,9 +256,9 @@ though neither Scrapy nor your project are leaking memory. This is due to a
(not so well) known problem of Python, which may not return released memory to
the operating system in some cases. For more information on this issue see:
* `Python Memory Management <http://www.evanjones.ca/python-memory.html>`_
* `Python Memory Management Part 2 <http://www.evanjones.ca/python-memory-part2.html>`_
* `Python Memory Management Part 3 <http://www.evanjones.ca/python-memory-part3.html>`_
* `Python Memory Management <https://www.evanjones.ca/python-memory.html>`_
* `Python Memory Management Part 2 <https://www.evanjones.ca/python-memory-part2.html>`_
* `Python Memory Management Part 3 <https://www.evanjones.ca/python-memory-part3.html>`_
The improvements proposed by Evan Jones, which are detailed in `this paper`_,
got merged in Python 2.5, but this only reduces the problem, it doesn't fix it
@ -327,7 +272,7 @@ completely. To quote the paper:
to move to a compacting garbage collector, which is able to move objects in
memory. This would require significant changes to the Python interpreter.*
.. _this paper: http://www.evanjones.ca/memoryallocator/
.. _this paper: https://www.evanjones.ca/memoryallocator/
To keep memory consumption reasonable you can split the job into several
smaller jobs or enable :ref:`persistent job queue <topics-jobs>`

12
docs/topics/link-extractors.rst
View File

@ -49,7 +49,7 @@ LxmlLinkExtractor
:type allow: a regular expression (or list of)
:param deny: a single regular expression (or list of regular expressions)
that the (absolute) urls must match in order to be excluded (ie. not
that the (absolute) urls must match in order to be excluded (i.e. not
extracted). It has precedence over the ``allow`` parameter. If not
given (or empty) it won't exclude any links.
:type deny: a regular expression (or list of)
@ -64,9 +64,13 @@ LxmlLinkExtractor
:param deny_extensions: a single value or list of strings containing
extensions that should be ignored when extracting links.
If not given, it will default to the
``IGNORED_EXTENSIONS`` list defined in the
`scrapy.linkextractors`_ package.
If not given, it will default to
:data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
.. versionchanged:: 2.0
:data:`~scrapy.linkextractors.IGNORED_EXTENSIONS` now includes
``7z``, ``7zip``, ``apk``, ``bz2``, ``cdr``, ``dmg``, ``ico``,
``iso``, ``tar``, ``tar.gz``, ``webm``, and ``xz``.
:type deny_extensions: list
:param restrict_xpaths: is an XPath (or list of XPath's) which defines

105
docs/topics/loaders.rst
View File

@ -7,13 +7,12 @@ Item Loaders
.. module:: scrapy.loader
:synopsis: Item Loader class
Item Loaders provide a convenient mechanism for populating scraped :ref:`Items
<topics-items>`. Even though Items can be populated using their own
dictionary-like API, Item Loaders provide a much more convenient API for
populating them from a scraping process, by automating some common tasks like
parsing the raw extracted data before assigning it.
Item Loaders provide a convenient mechanism for populating scraped :ref:`items
<topics-items>`. Even though items can be populated directly, Item Loaders provide a
much more convenient API for populating them from a scraping process, by automating
some common tasks like parsing the raw extracted data before assigning it.
In other words, :ref:`Items <topics-items>` provide the *container* of
In other words, :ref:`items <topics-items>` provide the *container* of
scraped data, while Item Loaders provide the mechanism for *populating* that
container.
@ -25,10 +24,10 @@ Using Item Loaders to populate items
====================================
To use an Item Loader, you must first instantiate it. You can either
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
which case an Item is automatically instantiated in the Item Loader ``__init__`` method
using the Item class specified in the :attr:`ItemLoader.default_item_class`
attribute.
instantiate it with an :ref:`item object <topics-items>` or without one, in which
case an :ref:`item object <topics-items>` is automatically created in the
Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
specified in the :attr:`ItemLoader.default_item_class` attribute.
Then, you start collecting values into the Item Loader, typically using
:ref:`Selectors <topics-selectors>`. You can add more than one value to
@ -77,6 +76,31 @@ called which actually returns the item populated with the data
previously extracted and collected with the :meth:`~ItemLoader.add_xpath`,
:meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls.
.. _topics-loaders-dataclass:
Working with dataclass items
============================
By default, :ref:`dataclass items <dataclass-items>` require all fields to be
passed when created. This could be an issue when using dataclass items with
item loaders: unless a pre-populated item is passed to the loader, fields
will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
One approach to overcome this is to define items using the
:func:`~dataclasses.field` function, with a ``default`` argument::
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class InventoryItem:
name: Optional[str] = field(default=None)
price: Optional[float] = field(default=None)
stock: Optional[int] = field(default=None)
.. _topics-loaders-processors:
Input and Output processors
@ -88,7 +112,7 @@ received (through the :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`
:meth:`~ItemLoader.add_value` methods) and the result of the input processor is
collected and kept inside the ItemLoader. After collecting all data, the
:meth:`ItemLoader.load_item` method is called to populate and get the populated
:class:`~scrapy.item.Item` object. That's when the output processor is
:ref:`item object <topics-items>`. That's when the output processor is
called with the data previously collected (and processed using the input
processor). The result of the output processor is the final value that gets
assigned to the item.
@ -136,6 +160,9 @@ with the data to be parsed, and return a parsed value. So you can use any
function as input or output processor. The only requirement is that they must
accept one (and only one) positional argument, which will be an iterable.
.. versionchanged:: 2.0
Processors no longer need to be methods.
.. note:: Both input and output processors must receive an iterable as their
first argument. The output of those functions can be anything. The result of
input processors will be appended to an internal list (in the Loader)
@ -150,12 +177,10 @@ Last, but not least, Scrapy comes with some :ref:`commonly used processors
<topics-loaders-available-processors>` built-in for convenience.
Declaring Item Loaders
======================
Item Loaders are declared like Items, by using a class definition syntax. Here
is an example::
Item Loaders are declared using a class definition syntax. Here is an example::
from scrapy.loader import ItemLoader
from scrapy.loader.processors import TakeFirst, MapCompose, Join
@ -270,11 +295,11 @@ There are several ways to modify Item Loader context values:
ItemLoader objects
==================
.. class:: ItemLoader([item, selector, response], \**kwargs)
.. class:: ItemLoader([item, selector, response], **kwargs)
Return a new Item Loader for populating the given Item. If no item is
given, one is instantiated automatically using the class in
:attr:`default_item_class`.
Return a new Item Loader for populating the given :ref:`item object
<topics-items>`. If no item object is given, one is instantiated
automatically using the class in :attr:`default_item_class`.
When instantiated with a ``selector`` or a ``response`` parameters
the :class:`ItemLoader` class provides convenient mechanisms for extracting
@ -283,7 +308,7 @@ ItemLoader objects
:param item: The item instance to populate using subsequent calls to
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
or :meth:`~ItemLoader.add_value`.
:type item: :class:`~scrapy.item.Item` object
:type item: :ref:`item object <topics-items>`
:param selector: The selector to extract data from, when using the
:meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
@ -300,7 +325,7 @@ ItemLoader objects
:class:`ItemLoader` instances have the following methods:
.. method:: get_value(value, \*processors, \**kwargs)
.. method:: get_value(value, *processors, **kwargs)
Process the given ``value`` by the given ``processors`` and keyword
arguments.
@ -318,7 +343,7 @@ ItemLoader objects
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
'FOO`
.. method:: add_value(field_name, value, \*processors, \**kwargs)
.. method:: add_value(field_name, value, *processors, **kwargs)
Process and then add the given ``value`` for the given field.
@ -340,11 +365,11 @@ ItemLoader objects
loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
loader.add_value(None, {'name': u'foo', 'sex': u'male'})
.. method:: replace_value(field_name, value, \*processors, \**kwargs)
.. method:: replace_value(field_name, value, *processors, **kwargs)
Similar to :meth:`add_value` but replaces the collected data with the
new value instead of adding it.
.. method:: get_xpath(xpath, \*processors, \**kwargs)
.. method:: get_xpath(xpath, *processors, **kwargs)
Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
@ -364,7 +389,7 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
.. method:: add_xpath(field_name, xpath, \*processors, \**kwargs)
.. method:: add_xpath(field_name, xpath, *processors, **kwargs)
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
value, which is used to extract a list of unicode strings from the
@ -382,12 +407,12 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
.. method:: replace_xpath(field_name, xpath, \*processors, \**kwargs)
.. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
Similar to :meth:`add_xpath` but replaces collected data instead of
adding it.
.. method:: get_css(css, \*processors, \**kwargs)
.. method:: get_css(css, *processors, **kwargs)
Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
@ -407,7 +432,7 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
.. method:: add_css(field_name, css, \*processors, \**kwargs)
.. method:: add_css(field_name, css, *processors, **kwargs)
Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
instead of a value, which is used to extract a list of unicode strings
@ -425,7 +450,7 @@ ItemLoader objects
# HTML snippet: <p id="price">the price is $1200</p>
loader.add_css('price', 'p#price', re='the price is (.*)')
.. method:: replace_css(field_name, css, \*processors, \**kwargs)
.. method:: replace_css(field_name, css, *processors, **kwargs)
Similar to :meth:`add_css` but replaces collected data instead of
adding it.
@ -441,17 +466,19 @@ ItemLoader objects
Create a nested loader with an xpath selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
object <topics-items>` with the parent :class:`ItemLoader` so calls to
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
behave as expected.
.. method:: nested_css(css)
Create a nested loader with a css selector.
The supplied selector is applied relative to selector associated
with this :class:`ItemLoader`. The nested loader shares the :class:`Item`
with the parent :class:`ItemLoader` so calls to :meth:`add_xpath`,
:meth:`add_value`, :meth:`replace_value`, etc. will behave as expected.
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
object <topics-items>` with the parent :class:`ItemLoader` so calls to
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
behave as expected.
.. method:: get_collected_values(field_name)
@ -474,7 +501,7 @@ ItemLoader objects
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
The :ref:`item object <topics-items>` being parsed by this Item Loader.
This is mostly used as a property so when attempting to override this
value, you may want to check out :attr:`default_item_class` first.
@ -485,8 +512,8 @@ ItemLoader objects
.. attribute:: default_item_class
An Item class (or factory), used to instantiate items when not given in
the ``__init__`` method.
An :ref:`item object <topics-items>` class or factory, used to
instantiate items when not given in the ``__init__`` method.
.. attribute:: default_input_processor
@ -675,7 +702,7 @@ Here is a list of all built-in processors:
>>> proc(['one', 'two', 'three'])
'one<br>two<br>three'
.. class:: Compose(\*functions, \**default_loader_context)
.. class:: Compose(*functions, **default_loader_context)
A processor which is constructed from the composition of the given
functions. This means that each input value of this processor is passed to
@ -703,7 +730,7 @@ Here is a list of all built-in processors:
active Loader context accessible through the :meth:`ItemLoader.context`
attribute.
.. class:: MapCompose(\*functions, \**default_loader_context)
.. class:: MapCompose(*functions, **default_loader_context)
A processor which is constructed from the composition of the given
functions, similar to the :class:`Compose` processor. The difference with

31
docs/topics/logging.rst
View File

@ -9,8 +9,7 @@ Logging
explicit calls to the Python standard logging. Keep reading to learn more
about the new logging system.
Scrapy uses `Python's builtin logging system
<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
Scrapy uses :mod:`logging` for event logging. We'll
provide some simple examples to get you started, but for more advanced
use-cases it's strongly suggested to read thoroughly its documentation.
@ -83,10 +82,10 @@ path::
.. seealso::
Module logging, `HowTo <https://docs.python.org/2/howto/logging.html>`_
Module logging, :doc:`HowTo <howto/logging>`
Basic Logging Tutorial
Module logging, `Loggers <https://docs.python.org/2/library/logging.html#logger-objects>`_
Module logging, :ref:`Loggers <logger>`
Further documentation on loggers
.. _topics-logging-from-spiders:
@ -165,14 +164,12 @@ possible levels listed in :ref:`topics-logging-levels`.
:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
used as layouts for all messages. Those strings can contain any placeholders
listed in `logging's logrecord attributes docs
<https://docs.python.org/2/library/logging.html#logrecord-attributes>`_ and
`datetime's strftime and strptime directives
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and
:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`
respectively.
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
component that prints the log. It is unset by default, hence logs contain the
component that prints the log. It is unset by default, hence logs contain the
Scrapy component responsible for that log output.
Command-line options
@ -190,7 +187,7 @@ to override some of the Scrapy settings regarding logging.
.. seealso::
Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
Module :mod:`logging.handlers`
Further documentation on available handlers
.. _custom-log-formats:
@ -201,10 +198,13 @@ Custom Log Formats
A custom log format can be set for different actions by extending
:class:`~scrapy.logformatter.LogFormatter` class and making
:setting:`LOG_FORMATTER` point to your new class.
.. autoclass:: scrapy.logformatter.LogFormatter
:members:
.. _topics-logging-advanced-customization:
Advanced customization
----------------------
@ -256,16 +256,15 @@ scrapy.utils.log module
In that case, its usage is not required but it's recommended.
Another option when running custom scripts is to manually configure the logging.
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
To do this you can use :func:`logging.basicConfig` to set a basic root handler.
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
so it is recommended to only use `logging.basicConfig()`_ together with
so it is recommended to only use :func:`logging.basicConfig` together with
:class:`~scrapy.crawler.CrawlerRunner`.
This is an example on how to redirect ``INFO`` or higher messages to a file::
import logging
from scrapy.utils.log import configure_logging
logging.basicConfig(
filename='log.txt',
@ -275,7 +274,3 @@ scrapy.utils.log module
Refer to :ref:`run-from-script` for more details about using Scrapy this
way.
.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig

73
docs/topics/media-pipeline.rst
View File

@ -50,7 +50,7 @@ this:
4. When the files are downloaded, another field (``files``) will be populated
with the results. This field will contain a list of dicts with information
about the downloaded files, such as the downloaded path, the original
scraped url (taken from the ``file_urls`` field) , and the file checksum.
scraped url (taken from the ``file_urls`` field), the file checksum and the file status.
The files in the list of the ``files`` field will retain the same order of
the original ``file_urls`` field. If some file failed downloading, an
error will be logged and the file won't be present in the ``files`` field.
@ -116,12 +116,6 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
Supported Storage
=================
File system is currently the only officially supported storage, but there are
also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
.. _Amazon S3: https://aws.amazon.com/s3/
.. _Google Cloud Storage: https://cloud.google.com/storage/
File system storage
-------------------
@ -147,9 +141,13 @@ Where:
* ``full`` is a sub-directory to separate full images from thumbnails (if
used). For more info see :ref:`topics-images-thumbnails`.
.. _media-pipeline-ftp:
FTP server storage
------------------
.. versionadded:: 2.0
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server.
Scrapy will automatically upload the files to the server.
@ -158,7 +156,7 @@ following forms::
ftp://username:password@address:port/path
ftp://address:port/path
If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
:setting:`FTP_PASSWORD` settings respectively.
@ -203,6 +201,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _media-pipeline-gcs:
Google Cloud Storage
---------------------
@ -245,20 +246,22 @@ Usage example
.. setting:: IMAGES_URLS_FIELD
.. setting:: IMAGES_RESULT_FIELD
In order to use a media pipeline first, :ref:`enable it
In order to use a media pipeline, first :ref:`enable it
<topics-media-pipeline-enabling>`.
Then, if a spider returns a dict with the URLs key (``file_urls`` or
``image_urls``, for the Files or Images Pipeline respectively), the pipeline will
put the results under respective key (``files`` or ``images``).
Then, if a spider returns an :ref:`item object <topics-items>` with the URLs
field (``file_urls`` or ``image_urls``, for the Files or Images Pipeline
respectively), the pipeline will put the results under the respective field
(``files`` or ``images``).
If you prefer to use :class:`~.Item`, then define a custom item with the
necessary fields, like in this example for Images Pipeline::
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.item.Item` class::
import scrapy
class MyItem(scrapy.Item):
# ... other item fields ...
image_urls = scrapy.Field()
images = scrapy.Field()
@ -410,7 +413,7 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
.. method:: file_path(request, response, info)
.. method:: file_path(self, request, response=None, info=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
@ -434,7 +437,7 @@ See here the methods that you can override in your custom Files Pipeline:
class MyFilesPipeline(FilesPipeline):
def file_path(self, request, response, info):
def file_path(self, request, response=None, info=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
By default the :meth:`file_path` method returns
@ -447,8 +450,11 @@ See here the methods that you can override in your custom Files Pipeline:
:meth:`~get_media_requests` method and return a Request for each
file URL::
from itemadapter import ItemAdapter
def get_media_requests(self, item, info):
for file_url in item['file_urls']:
adapter = ItemAdapter(item)
for file_url in adapter['file_urls']:
yield scrapy.Request(file_url)
Those requests will be processed by the pipeline and, when they have finished
@ -472,6 +478,18 @@ See here the methods that you can override in your custom Files Pipeline:
* ``checksum`` - a `MD5 hash`_ of the image contents
* ``status`` - the file status indication.
.. versionadded:: 2.2
It can be one of the following:
* ``downloaded`` - file was downloaded.
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
according to the file expiration policy.
* ``cached`` - file was already scheduled for download, by another item
sharing the same file.
The list of tuples received by :meth:`~item_completed` is
guaranteed to retain the same order of the requests returned from the
:meth:`~get_media_requests` method.
@ -481,7 +499,8 @@ See here the methods that you can override in your custom Files Pipeline:
[(True,
{'checksum': '2b00042f7481c7b056c4b410d28f33cf',
'path': 'full/0a79c461a4062ac383dc4fade7bc09f1384a3910.jpg',
'url': 'http://www.example.com/files/product1.pdf'}),
'url': 'http://www.example.com/files/product1.pdf',
'status': 'downloaded'}),
(False,
Failure(...))]
@ -502,13 +521,15 @@ See here the methods that you can override in your custom Files Pipeline:
store the downloaded file paths (passed in results) in the ``file_paths``
item field, and we drop the item if it doesn't contain any files::
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
def item_completed(self, results, item, info):
file_paths = [x['path'] for ok, x in results if ok]
if not file_paths:
raise DropItem("Item contains no files")
item['file_paths'] = file_paths
adapter = ItemAdapter(item)
adapter['file_paths'] = file_paths
return item
By default, the :meth:`item_completed` method returns the item.
@ -524,7 +545,7 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
.. method:: file_path(request, response, info)
.. method:: file_path(self, request, response=None, info=None)
This method is called once per downloaded item. It returns the
download path of the file originating from the specified
@ -548,7 +569,7 @@ See here the methods that you can override in your custom Images Pipeline:
class MyImagesPipeline(ImagesPipeline):
def file_path(self, request, response, info):
def file_path(self, request, response=None, info=None):
return 'files/' + os.path.basename(urlparse(request.url).path)
By default the :meth:`file_path` method returns
@ -573,6 +594,8 @@ See here the methods that you can override in your custom Images Pipeline:
By default, the :meth:`item_completed` method returns the item.
.. _media-pipeline-example:
Custom Images pipeline example
==============================
@ -580,8 +603,9 @@ Here is a full example of the Images Pipeline whose methods are exemplified
above::
import scrapy
from scrapy.pipelines.images import ImagesPipeline
from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline
class MyImagesPipeline(ImagesPipeline):
@ -593,7 +617,8 @@ above::
image_paths = [x['path'] for ok, x in results if ok]
if not image_paths:
raise DropItem("Item contains no images")
item['image_paths'] = image_paths
adapter = ItemAdapter(item)
adapter['image_paths'] = image_paths
return item

5
docs/topics/practices.rst
View File

@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it.
...
process = CrawlerProcess(settings={
'FEED_FORMAT': 'json',
'FEED_URI': 'items.json'
"FEEDS": {
"items.json": {"format": "json"},
},
})
process.crawl(MySpider)

158
docs/topics/request-response.rst
View File

@ -31,10 +31,12 @@ Request objects
a :class:`Response`.
:param url: the URL of this request
If the URL is invalid, a :exc:`ValueError` exception is raised.
:type url: string
:param callback: the function that will be called with the response of this
request (once its downloaded) as its first parameter. For more information
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.spiders.Spider.parse` method will be used.
@ -125,6 +127,10 @@ Request objects
:exc:`~twisted.python.failure.Failure` as first parameter.
For more information,
see :ref:`topics-request-response-ref-errbacks` below.
.. versionchanged:: 2.0
The *callback* parameter is no longer required when the *errback*
parameter is specified.
:type errback: callable
:param flags: Flags sent to the request, can be used for logging or similar purposes.
@ -168,9 +174,9 @@ Request objects
See :ref:`topics-request-meta` for a list of special meta keys
recognized by Scrapy.
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.meta`` attribute.
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.meta`` attribute.
.. attribute:: Request.cb_kwargs
@ -179,11 +185,13 @@ Request objects
for new Requests, which means by default callbacks only get a :class:`Response`
object as argument.
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.cb_kwargs`` attribute.
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.cb_kwargs`` attribute.
.. _shallow copied: https://docs.python.org/2/library/copy.html
In case of a failure to process the request, this dict can be accessed as
``failure.request.cb_kwargs`` in the request's errback. For more information,
see :ref:`errback-cb_kwargs`.
.. method:: Request.copy()
@ -308,6 +316,31 @@ errors if needed::
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
.. _errback-cb_kwargs:
Accessing additional data in errback functions
----------------------------------------------
In case of a failure to process the request, you may be interested in
accessing arguments to the callback functions so you can process further
based on the arguments in the errback. The following example shows how to
achieve this by using ``Failure.request.cb_kwargs``::
def parse(self, response):
request = scrapy.Request('http://www.example.com/index.html',
callback=self.parse_page2,
errback=self.errback_page2,
cb_kwargs=dict(main_url=response.url))
yield request
def parse_page2(self, response, main_url):
pass
def errback_page2(self, failure):
yield dict(
main_url=failure.request.cb_kwargs['main_url'],
)
.. _topics-request-meta:
Request.meta special keys
@ -381,6 +414,51 @@ The meta key is used set retry times per request. When initialized, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
.. _topics-stop-response-download:
Stopping the download of a Response
===================================
Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
:class:`~scrapy.signals.bytes_received` signal handler will stop the
download of a given response. See the following example::
import scrapy
class StopSpider(scrapy.Spider):
name = "stop"
start_urls = ["https://docs.scrapy.org/en/latest/"]
@classmethod
def from_crawler(cls, crawler):
spider = super().from_crawler(crawler)
crawler.signals.connect(spider.on_bytes_received, signal=scrapy.signals.bytes_received)
return spider
def parse(self, response):
# 'last_chars' show that the full response was not downloaded
yield {"len": len(response.text), "last_chars": response.text[-40:]}
def on_bytes_received(self, data, request, spider):
raise scrapy.exceptions.StopDownload(fail=False)
which produces the following output::
2020-05-19 17:26:12 [scrapy.core.engine] INFO: Spider opened
2020-05-19 17:26:12 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2020-05-19 17:26:13 [scrapy.core.downloader.handlers.http11] DEBUG: Download stopped for <GET https://docs.scrapy.org/en/latest/> from signal handler StopSpider.on_bytes_received
2020-05-19 17:26:13 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://docs.scrapy.org/en/latest/> (referer: None) ['download_stopped']
2020-05-19 17:26:13 [scrapy.core.scraper] DEBUG: Scraped from <200 https://docs.scrapy.org/en/latest/>
{'len': 279, 'last_chars': 'dth, initial-scale=1.0">\n \n <title>Scr'}
2020-05-19 17:26:13 [scrapy.core.engine] INFO: Closing spider (finished)
By default, resulting responses are handled by their corresponding errbacks. To
call their callback instead, like in this example, pass ``fail=False`` to the
:exc:`~scrapy.exceptions.StopDownload` exception.
.. _topics-request-response-ref-request-subclasses:
Request subclasses
@ -396,7 +474,7 @@ The FormRequest class extends the base :class:`Request` with functionality for
dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form
fields with form data from :class:`Response` objects.
.. _lxml.html forms: http://lxml.de/lxmlhtml.html#forms
.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
.. class:: FormRequest(url, [formdata, ...])
@ -560,12 +638,10 @@ dealing with JSON requests.
set to ``'POST'`` automatically.
:type data: JSON serializable object
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
data into JSON format.
:type dumps_kwargs: dict
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
JsonRequest usage example
-------------------------
@ -609,7 +685,16 @@ Response objects
:param request: the initial value of the :attr:`Response.request` attribute.
This represents the :class:`Request` that generated this response.
:type request: :class:`Request` object
:type request: scrapy.http.Request
:param certificate: an object representing the server's SSL certificate.
:type certificate: twisted.internet.ssl.Certificate
:param ip_address: The IP address of the server from which the Response originated.
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
.. versionadded:: 2.1.0
The ``ip_address`` parameter.
.. attribute:: Response.url
@ -664,7 +749,7 @@ Response objects
.. attribute:: Response.meta
A shortcut to the :attr:`Request.meta` attribute of the
:attr:`Response.request` object (ie. ``self.request.meta``).
:attr:`Response.request` object (i.e. ``self.request.meta``).
Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta`
attribute is propagated along redirects and retries, so you will get
@ -672,6 +757,20 @@ Response objects
.. seealso:: :attr:`Request.meta` attribute
.. attribute:: Response.cb_kwargs
.. versionadded:: 2.0
A shortcut to the :attr:`Request.cb_kwargs` attribute of the
:attr:`Response.request` object (i.e. ``self.request.cb_kwargs``).
Unlike the :attr:`Response.request` attribute, the
:attr:`Response.cb_kwargs` attribute is propagated along redirects and
retries, so you will get the original :attr:`Request.cb_kwargs` sent
from your spider.
.. seealso:: :attr:`Request.cb_kwargs` attribute
.. attribute:: Response.flags
A list that contains flags for this response. Flags are labels used for
@ -679,6 +778,23 @@ Response objects
they're shown on the string representation of the Response (`__str__`
method) which is used by the engine for logging.
.. attribute:: Response.certificate
A :class:`twisted.internet.ssl.Certificate` object representing
the server's SSL certificate.
Only populated for ``https`` responses, ``None`` otherwise.
.. attribute:: Response.ip_address
.. versionadded:: 2.1.0
The IP address of the server from which the Response originated.
This attribute is currently only populated by the HTTP 1.1 download
handler, i.e. for ``http(s)`` responses. For other handlers,
:attr:`ip_address` is always ``None``.
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.
@ -694,18 +810,16 @@ Response objects
Constructs an absolute url by combining the Response's :attr:`url` with
a possible relative url.
This is a wrapper over `urlparse.urljoin`_, it's merely an alias for
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
making this call::
urlparse.urljoin(response.url, url)
urllib.parse.urljoin(response.url, url)
.. automethod:: Response.follow
.. automethod:: Response.follow_all
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
.. _topics-request-response-ref-response-subclasses:
Response subclasses
@ -760,7 +874,7 @@ TextResponse objects
1. the encoding passed in the ``__init__`` method ``encoding`` argument
2. the encoding declared in the Content-Type HTTP header. If this
encoding is not valid (ie. unknown), it is ignored and the next
encoding is not valid (i.e. unknown), it is ignored and the next
resolution mechanism is tried.
3. the encoding declared in the response body. The TextResponse class
@ -794,10 +908,10 @@ TextResponse objects
.. automethod:: TextResponse.follow_all
.. method:: TextResponse.body_as_unicode()
.. automethod:: TextResponse.json()
The same as :attr:`text`, but available as a method. This method is
kept for backward compatibility; please prefer ``response.text``.
Returns a Python object from deserialized JSON document.
The result is cached after the first call.
HtmlResponse objects

24
docs/topics/selectors.rst
View File

@ -14,7 +14,7 @@ achieve this, such as:
drawback: it's slow.
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on `ElementTree`_. (lxml is not part of the Python standard
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
library.)
Scrapy comes with its own mechanism for extracting data. They're called
@ -35,12 +35,10 @@ defines selectors to associate those styles with specific HTML elements.
in speed and parsing accuracy to lxml.
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: http://lxml.de/
.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
.. _cssselect: https://pypi.python.org/pypi/cssselect/
.. _XPath: https://www.w3.org/TR/xpath
.. _lxml: https://lxml.de/
.. _XPath: https://www.w3.org/TR/xpath/all/
.. _CSS: https://www.w3.org/TR/selectors
.. _parsel: https://parsel.readthedocs.io/
.. _parsel: https://parsel.readthedocs.io/en/latest/
Using selectors
===============
@ -255,7 +253,7 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
They will most probably not work with other libraries like
`lxml`_ or `PyQuery`_.
.. _PyQuery: https://pypi.python.org/pypi/pyquery
.. _PyQuery: https://pypi.org/project/pyquery/
Examples:
@ -309,7 +307,7 @@ Examples:
make much sense: text nodes do not have attributes, and attribute values
are string values already and do not have children nodes.
.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
.. _CSS Selectors: https://www.w3.org/TR/selectors-3/#selectors
.. _topics-selectors-nesting-selectors:
@ -504,7 +502,7 @@ Another common case would be to extract all direct ``<p>`` children:
For more details about relative XPaths see the `Location Paths`_ section in the
XPath specification.
.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
.. _Location Paths: https://www.w3.org/TR/xpath/all/#location-paths
When querying by class, consider using CSS
------------------------------------------
@ -612,7 +610,7 @@ But using the ``.`` to mean the node, works:
>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
.. _`XPath string function`: https://www.w3.org/TR/xpath/all/#section-String-Functions
.. _topics-selectors-xpath-variables:
@ -764,7 +762,7 @@ Set operations
These can be handy for excluding parts of a document tree before
extracting text elements for example.
Example extracting microdata (sample content taken from http://schema.org/Product)
Example extracting microdata (sample content taken from https://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
>>> doc = u"""
@ -986,7 +984,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
sel = Selector(html_response)
1. Select all ``<h1>`` elements from an HTML response body, returning a list of
:class:`Selector` objects (ie. a :class:`SelectorList` object)::
:class:`Selector` objects (i.e. a :class:`SelectorList` object)::
sel.xpath("//h1")
@ -1013,7 +1011,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object::
sel = Selector(xml_response)
1. Select all ``<product>`` elements from an XML response body, returning a list
of :class:`Selector` objects (ie. a :class:`SelectorList` object)::
of :class:`Selector` objects (i.e. a :class:`SelectorList` object)::
sel.xpath("//product")

191
docs/topics/settings.rst
View File

@ -26,9 +26,7 @@ do this by using an environment variable, ``SCRAPY_SETTINGS_MODULE``.
The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
``myproject.settings``. Note that the settings module should be on the
Python `import search path`_.
.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
Python :ref:`import search path <tut-searchpath>`.
.. _populating-settings:
@ -124,7 +122,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
attribute of the Crawler that is passed to ``from_crawler`` method in
extensions, middlewares and item pipelines::
class MyExtension(object):
class MyExtension:
def __init__(self, log_is_enabled=False):
if log_is_enabled:
print("log is enabled!")
@ -160,27 +158,6 @@ to any particular component. In that case the module of that component will be
shown, typically an extension, middleware or pipeline. It also means that the
component must be enabled in order for the setting to have any effect.
.. setting:: ASYNCIO_REACTOR
ASYNCIO_REACTOR
---------------
Default: ``False``
Whether to install and require the Twisted reactor that uses the asyncio loop.
When this option is set to ``True``, Scrapy will require
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
install this reactor if no reactor is installed yet, such as when using the
``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
manually. If a different reactor is installed outside Scrapy, it will raise an
exception.
The default value for this option is currently ``False`` to maintain backward
compatibility and avoid possible problems caused by using a different Twisted
reactor.
.. setting:: AWS_ACCESS_KEY_ID
AWS_ACCESS_KEY_ID
@ -259,8 +236,8 @@ CONCURRENT_ITEMS
Default: ``100``
Maximum number of concurrent items (per response) to process in parallel in the
Item Processor (also known as the :ref:`Item Pipeline <topics-item-pipeline>`).
Maximum number of concurrent items (per response) to process in parallel in
:ref:`item pipelines <topics-item-pipeline>`.
.. setting:: CONCURRENT_REQUESTS
@ -269,7 +246,7 @@ CONCURRENT_REQUESTS
Default: ``16``
The maximum number of concurrent (ie. simultaneous) requests that will be
The maximum number of concurrent (i.e. simultaneous) requests that will be
performed by the Scrapy downloader.
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
@ -279,7 +256,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN
Default: ``8``
The maximum number of concurrent (ie. simultaneous) requests that will be
The maximum number of concurrent (i.e. simultaneous) requests that will be
performed to any single domain.
See also: :ref:`topics-autothrottle` and its
@ -293,7 +270,7 @@ CONCURRENT_REQUESTS_PER_IP
Default: ``0``
The maximum number of concurrent (ie. simultaneous) requests that will be
The maximum number of concurrent (i.e. simultaneous) requests that will be
performed to any single IP. If non-zero, the
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is
used instead. In other words, concurrency limits will be applied per IP, not
@ -402,6 +379,8 @@ DNS in-memory cache size.
DNS_RESOLVER
------------
.. versionadded:: 2.0
Default: ``'scrapy.resolver.CachingThreadedResolver'``
The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver``
@ -441,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
.. note::
HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
unless you use Twisted<11.1, or if you really want to use HTTP/1.0
and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
accordingly, i.e. to
``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
unless you really want to use HTTP/1.0 and override
:setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
@ -468,7 +446,6 @@ or even enable client-side authentication (and various other things).
Scrapy also has another context factory class that you can set,
``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
which uses the platform's certificates to validate remote endpoints.
**This is only available if you use Twisted>=14.0.**
If you do use a custom ContextFactory, make sure its ``__init__`` method
accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
@ -492,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
specific cipher that is not included in ``DEFAULT`` if a website requires it.
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
@ -515,10 +492,6 @@ This setting must be one of these string values:
- ``'TLSv1.2'``: forces TLS version 1.2
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
.. note::
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
or above (Twisted>=14.0 if you can).
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@ -681,8 +654,6 @@ If you want to disable it set to 0.
spider attribute and per-request using :reqmeta:`download_maxsize`
Request.meta key.
This feature needs Twisted >= 11.1.
.. setting:: DOWNLOAD_WARNSIZE
DOWNLOAD_WARNSIZE
@ -700,8 +671,6 @@ If you want to disable it set to 0.
spider attribute and per-request using :reqmeta:`download_warnsize`
Request.meta key.
This feature needs Twisted >= 11.1.
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
DOWNLOAD_FAIL_ON_DATALOSS
@ -918,10 +887,9 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
placeholders.
.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
list of available placeholders.
.. setting:: LOG_DATEFORMAT
@ -931,10 +899,9 @@ LOG_DATEFORMAT
Default: ``'%Y-%m-%d %H:%M:%S'``
String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
directives.
.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
in :setting:`LOG_FORMAT`. Refer to the
:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
whole list of available directives.
.. setting:: LOG_FORMATTER
@ -1135,17 +1102,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
this value if you're experiencing problems with insufficient blocking IO.
.. setting:: REDIRECT_MAX_TIMES
REDIRECT_MAX_TIMES
------------------
Default: ``20``
Defines the maximum times a request can be redirected. After this maximum the
request's response is returned as is. We used Firefox default value for the
same task.
.. setting:: REDIRECT_PRIORITY_ADJUST
REDIRECT_PRIORITY_ADJUST
@ -1279,6 +1235,9 @@ does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
SCRAPER_SLOT_MAX_ACTIVE_SIZE
----------------------------
.. versionadded:: 2.0
Default: ``5_000_000``
Soft limit (in bytes) for response data being processed.
@ -1438,17 +1397,6 @@ Default: ``True``
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
will be enabled (provided its extension is also enabled).
.. setting:: TELNETCONSOLE_PORT
TELNETCONSOLE_PORT
------------------
Default: ``[6023, 6073]``
The port range to use for the telnet console. If set to ``None`` or ``0``, a
dynamically assigned port is used. For more info see
:ref:`topics-telnetconsole`.
.. setting:: TEMPLATES_DIR
TEMPLATES_DIR
@ -1463,6 +1411,101 @@ command.
The project name must not conflict with the name of custom files or directories
in the ``project`` subdirectory.
.. setting:: TWISTED_REACTOR
TWISTED_REACTOR
---------------
.. versionadded:: 2.0
Default: ``None``
Import path of a given :mod:`~twisted.internet.reactor`.
Scrapy will install this reactor if no other reactor is installed yet, such as
when the ``scrapy`` CLI program is invoked or when using the
:class:`~scrapy.crawler.CrawlerProcess` class.
If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also
need to install the correct reactor manually. You can do that using
:func:`~scrapy.utils.reactor.install_reactor`:
.. autofunction:: scrapy.utils.reactor.install_reactor
If a reactor is already installed,
:func:`~scrapy.utils.reactor.install_reactor` has no effect.
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
:exc:`Exception` if the installed reactor does not match the
:setting:`TWISTED_REACTOR` setting; therfore, having top-level
:mod:`~twisted.internet.reactor` imports in project files and imported
third-party libraries will make Scrapy raise :exc:`Exception` when
it checks which reactor is installed.
In order to use the reactor installed by Scrapy::
import scrapy
from twisted.internet import reactor
class QuotesSpider(scrapy.Spider):
name = 'quotes'
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
which raises :exc:`Exception`, becomes::
import scrapy
class QuotesSpider(scrapy.Spider):
name = 'quotes'
def __init__(self, *args, **kwargs):
self.timeout = int(kwargs.pop('timeout', '60'))
super(QuotesSpider, self).__init__(*args, **kwargs)
def start_requests(self):
from twisted.internet import reactor
reactor.callLater(self.timeout, self.stop)
urls = ['http://quotes.toscrape.com/page/1']
for url in urls:
yield scrapy.Request(url=url, callback=self.parse)
def parse(self, response):
for quote in response.css('div.quote'):
yield {'text': quote.css('span.text::text').get()}
def stop(self):
self.crawler.engine.close_spider(self, 'timeout')
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will not attempt to install any specific reactor, and the
default reactor defined by Twisted for the current platform will be used. This
is to maintain backward compatibility and avoid possible problems caused by
using a non-default reactor.
For additional information, see :doc:`core/howto/choosing-reactor`.
.. setting:: URLLENGTH_LIMIT

17
docs/topics/shell.rst
View File

@ -41,7 +41,7 @@ variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
.. _IPython: https://ipython.org/
.. _IPython installation guide: https://ipython.org/install.html
.. _bpython: https://www.bpython-interpreter.org/
.. _bpython: https://bpython-interpreter.org/
Launch the shell
================
@ -142,7 +142,7 @@ Example of shell session
========================
Here's an example of a typical shell session where we start by scraping the
https://scrapy.org page, and then proceed to scrape the https://reddit.com
https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/
page. Finally, we modify the (Reddit) request method to POST and re-fetch it
getting an error. We end the session by typing Ctrl-D (in Unix systems) or
Ctrl-Z in Windows.
@ -156,6 +156,17 @@ First, we launch the shell::
scrapy shell 'https://scrapy.org' --nolog
.. note::
Remember to always enclose URLs in quotes when running the Scrapy shell from
the command line, otherwise URLs containing arguments (i.e. the ``&`` character)
will not work.
On Windows, use double quotes instead::
scrapy shell "https://scrapy.org" --nolog
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
list of available objects and useful shortcuts (you'll notice that these lines
all start with the ``[s]`` prefix)::
@ -182,7 +193,7 @@ After that, we can start playing with the objects:
>>> response.xpath('//title/text()').get()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> fetch("https://reddit.com")
>>> fetch("https://old.reddit.com/")
>>> response.xpath('//title/text()').get()
'reddit: the front page of the internet'

186
docs/topics/signals.rst
View File

@ -16,8 +16,7 @@ deliver the arguments that the handler receives.
You can connect to signals (or send your own) through the
:ref:`topics-api-signals`.
Here is a simple example showing how you can catch signals and perform some action:
::
Here is a simple example showing how you can catch signals and perform some action::
from scrapy import signals
from scrapy import Spider
@ -46,14 +45,51 @@ Here is a simple example showing how you can catch signals and perform some acti
def parse(self, response):
pass
.. _signal-deferred:
Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, see the :ref:`topics-signals-ref` below to know
which ones.
objects from their handlers, allowing you to run asynchronous code that
does not block Scrapy. If a signal handler returns a
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
:class:`~twisted.internet.defer.Deferred` to fire.
Let's take an example::
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['http://quotes.toscrape.com/page/1/']
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
def item_scraped(self, item):
# Send the scraped item to the server
d = treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
# The next item will be scraped only after
# deferred (d) is fired
return d
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
See the :ref:`topics-signals-ref` below to know which signals support
:class:`~twisted.internet.defer.Deferred`.
.. _topics-signals-ref:
@ -65,22 +101,25 @@ Built-in signals reference
Here's the list of Scrapy built-in signals and their meaning.
engine_started
Engine signals
--------------
engine_started
~~~~~~~~~~~~~~
.. signal:: engine_started
.. function:: engine_started()
Sent when the Scrapy engine has started crawling.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
.. note:: This signal may be fired *after* the :signal:`spider_opened` signal,
depending on how the spider was started. So **don't** rely on this signal
getting fired before :signal:`spider_opened`.
engine_stopped
--------------
~~~~~~~~~~~~~~
.. signal:: engine_stopped
.. function:: engine_stopped()
@ -88,10 +127,21 @@ engine_stopped
Sent when the Scrapy engine is stopped (for example, when a crawling
process has finished).
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
Item signals
------------
.. note::
As at max :setting:`CONCURRENT_ITEMS` items are processed in
parallel, many deferreds are fired together using
:class:`~twisted.internet.defer.DeferredList`. Hence the next
batch waits for the :class:`~twisted.internet.defer.DeferredList`
to fire and then runs the respective item signal handler for
the next batch of scraped items.
item_scraped
------------
~~~~~~~~~~~~
.. signal:: item_scraped
.. function:: item_scraped(item, response, spider)
@ -99,10 +149,10 @@ item_scraped
Sent when an item has been scraped, after it has passed all the
:ref:`topics-item-pipeline` stages (without being dropped).
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item scraped
:type item: dict or :class:`~scrapy.item.Item` object
:param item: the scraped item
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
@ -111,7 +161,7 @@ item_scraped
:type response: :class:`~scrapy.http.Response` object
item_dropped
------------
~~~~~~~~~~~~
.. signal:: item_dropped
.. function:: item_dropped(item, response, exception, spider)
@ -119,10 +169,10 @@ item_dropped
Sent after an item has been dropped from the :ref:`topics-item-pipeline`
when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spiders.Spider` object
@ -136,18 +186,18 @@ item_dropped
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
item_error
------------
~~~~~~~~~~
.. signal:: item_error
.. function:: item_error(item, response, spider, failure)
Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param item: the item dropped from the :ref:`topics-item-pipeline`
:type item: dict or :class:`~scrapy.item.Item` object
:param item: the item that caused the error in the :ref:`topics-item-pipeline`
:type item: :ref:`item object <item-types>`
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
@ -158,8 +208,11 @@ item_error
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
Spider signals
--------------
spider_closed
-------------
~~~~~~~~~~~~~
.. signal:: spider_closed
.. function:: spider_closed(spider, reason)
@ -167,7 +220,7 @@ spider_closed
Sent after a spider has been closed. This can be used to release per-spider
resources reserved on :signal:`spider_opened`.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been closed
:type spider: :class:`~scrapy.spiders.Spider` object
@ -182,7 +235,7 @@ spider_closed
:type reason: str
spider_opened
-------------
~~~~~~~~~~~~~
.. signal:: spider_opened
.. function:: spider_opened(spider)
@ -191,13 +244,13 @@ spider_opened
reserve per-spider resources, but can be used for any task that needs to be
performed when a spider is opened.
This signal supports returning deferreds from their handlers.
This signal supports returning deferreds from its handlers.
:param spider: the spider which has been opened
:type spider: :class:`~scrapy.spiders.Spider` object
spider_idle
-----------
~~~~~~~~~~~
.. signal:: spider_idle
.. function:: spider_idle(spider)
@ -215,7 +268,7 @@ spider_idle
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
prevent the spider from being closed.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param spider: the spider which has gone idle
:type spider: :class:`~scrapy.spiders.Spider` object
@ -227,14 +280,14 @@ spider_idle
due to duplication).
spider_error
------------
~~~~~~~~~~~~
.. signal:: spider_error
.. function:: spider_error(failure, response, spider)
Sent when a spider callback generates an error (ie. raises an exception).
Sent when a spider callback generates an error (i.e. raises an exception).
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
@ -245,8 +298,11 @@ spider_error
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
Request signals
---------------
request_scheduled
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: request_scheduled
.. function:: request_scheduled(request, spider)
@ -254,7 +310,7 @@ request_scheduled
Sent when the engine schedules a :class:`~scrapy.http.Request`, to be
downloaded later.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
@ -263,7 +319,7 @@ request_scheduled
:type spider: :class:`~scrapy.spiders.Spider` object
request_dropped
---------------
~~~~~~~~~~~~~~~
.. signal:: request_dropped
.. function:: request_dropped(request, spider)
@ -271,7 +327,7 @@ request_dropped
Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be
downloaded later, is rejected by the scheduler.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the scheduler
:type request: :class:`~scrapy.http.Request` object
@ -280,14 +336,14 @@ request_dropped
:type spider: :class:`~scrapy.spiders.Spider` object
request_reached_downloader
---------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_reached_downloader
.. function:: request_reached_downloader(request, spider)
Sent when a :class:`~scrapy.http.Request` reached downloader.
The signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached downloader
:type request: :class:`~scrapy.http.Request` object
@ -295,8 +351,60 @@ request_reached_downloader
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
request_left_downloader
~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_left_downloader
.. function:: request_left_downloader(request, spider)
.. versionadded:: 2.0
Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
failure.
This signal does not support returning deferreds from its handlers.
:param request: the request that reached the downloader
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
bytes_received
~~~~~~~~~~~~~~
.. versionadded:: 2.2
.. signal:: bytes_received
.. function:: bytes_received(data, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
received for a specific request. This signal might be fired multiple
times for the same request, with partial data each time. For instance,
a possible scenario for a 25 kb response would be two signals fired
with 10 kb of data, and a final one with 5 kb of data.
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
:type data: :class:`bytes` object
:param request: the request that generated the download
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: Handlers of this signal can stop the download of a response while it
is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
exception. Please refer to the :ref:`topics-stop-response-download` topic
for additional information and examples.
Response signals
----------------
response_received
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: response_received
.. function:: response_received(response, request, spider)
@ -304,7 +412,7 @@ response_received
Sent when the engine receives a new :class:`~scrapy.http.Response` from the
downloader.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param response: the response received
:type response: :class:`~scrapy.http.Response` object
@ -316,14 +424,14 @@ response_received
:type spider: :class:`~scrapy.spiders.Spider` object
response_downloaded
-------------------
~~~~~~~~~~~~~~~~~~~
.. signal:: response_downloaded
.. function:: response_downloaded(response, request, spider)
Sent by the downloader right after a ``HTTPResponse`` is downloaded.
This signal does not support returning deferreds from their handlers.
This signal does not support returning deferreds from its handlers.
:param response: the response downloaded
:type response: :class:`~scrapy.http.Response` object

23
docs/topics/spider-middleware.rst
View File

@ -102,29 +102,28 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
objects.
:class:`~scrapy.http.Request` objects and :ref:`item object
<topics-items>`.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
:param result: the result returned by the spider
:type result: an iterable of :class:`~scrapy.http.Request`, dict
or :class:`~scrapy.item.Item` objects
:type result: an iterable of :class:`~scrapy.http.Request` objects and
:ref:`item object <topics-items>`
:param spider: the spider whose result is being processed
:type spider: :class:`~scrapy.spiders.Spider` object
.. method:: process_spider_exception(response, exception, spider)
This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
iterable of :class:`~scrapy.http.Request`, dict or
:class:`~scrapy.item.Item` objects.
iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
<topics-items>`.
If it returns ``None``, Scrapy will continue processing this exception,
executing any other :meth:`process_spider_exception` in the following
@ -140,7 +139,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type response: :class:`~scrapy.http.Response` object
:param exception: the exception raised
:type exception: `Exception`_ object
:type exception: :exc:`Exception` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
@ -173,20 +172,16 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type spider: :class:`~scrapy.spiders.Spider` object
.. method:: from_crawler(cls, crawler)
If present, this classmethod is called to create a middleware instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the middleware. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for middleware to
access them and hook its functionality into Scrapy.
:param crawler: crawler that uses this middleware
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
.. _topics-spider-middleware-ref:
Built-in spider middleware reference

29
docs/topics/spiders.rst
View File

@ -23,8 +23,8 @@ For spiders, the scraping cycle goes through something like this:
:attr:`~scrapy.spiders.Spider.parse` method as callback function for the
Requests.
2. In the callback function, you parse the response (web page) and return either
dicts with extracted data, :class:`~scrapy.item.Item` objects,
2. In the callback function, you parse the response (web page) and return
:ref:`item objects <topics-items>`,
:class:`~scrapy.http.Request` objects, or an iterable of these objects.
Those Requests will also contain a callback (maybe
the same) and will then be downloaded by Scrapy and then their
@ -121,7 +121,7 @@ scrapy.Spider
send log messages through it as described on
:ref:`topics-logging-from-spiders`.
.. method:: from_crawler(crawler, \*args, \**kwargs)
.. method:: from_crawler(crawler, *args, **kwargs)
This is the class method used by Scrapy to create your spiders.
@ -179,8 +179,8 @@ scrapy.Spider
the same requirements as the :class:`Spider` class.
This method, as well as any other Request callback, must return an
iterable of :class:`~scrapy.http.Request` and/or
dicts or :class:`~scrapy.item.Item` objects.
iterable of :class:`~scrapy.http.Request` and/or :ref:`item objects
<topics-items>`.
:param response: the response to parse
:type response: :class:`~scrapy.http.Response`
@ -234,7 +234,7 @@ Return multiple Requests and items from a single callback::
yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :ref:`topics-items`::
to give data more structure you can use :class:`~scrapy.item.Item` objects::
import scrapy
from myproject.items import MyItem
@ -298,9 +298,7 @@ Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
If you were to set the ``start_urls`` attribute from the command line,
you would have to parse it on your own into a list
using something like
`ast.literal_eval <https://docs.python.org/library/ast.html#ast.literal_eval>`_
or `json.loads <https://docs.python.org/library/json.html#json.loads>`_
using something like :func:`ast.literal_eval` or :func:`json.loads`
and then set it as an attribute.
Otherwise, you would cause iteration over a ``start_urls`` string
(a very common python pitfall)
@ -366,7 +364,7 @@ CrawlSpider
This method is called for the start_urls responses. It allows to parse
the initial responses and must return either an
:class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
object, or an iterable containing any of them.
Crawling rules
@ -385,7 +383,7 @@ Crawling rules
object with that name will be used) to be called for each link extracted with
the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
as its first argument and must return either a single instance or an iterable of
:class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
:ref:`item objects <topics-items>` and/or :class:`~scrapy.http.Request` objects
(or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
in its ``meta`` dictionary (under the ``link_text`` key)
@ -420,6 +418,9 @@ Crawling rules
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
.. versionadded:: 2.0
The *errback* parameter.
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
@ -530,7 +531,7 @@ XMLFeedSpider
(``itertag``). Receives the response and an
:class:`~scrapy.selector.Selector` for each node. Overriding this
method is mandatory. Otherwise, you spider won't work. This method
must return either a :class:`~scrapy.item.Item` object, a
must return an :ref:`item object <topics-items>`, a
:class:`~scrapy.http.Request` object, or an iterable containing any of
them.
@ -540,7 +541,7 @@ XMLFeedSpider
spider, and it's intended to perform any last time processing required
before returning the results to the framework core, for example setting the
item IDs. It receives a list of results and the response which originated
those results. It must return a list of results (Items or Requests).
those results. It must return a list of results (items or requests).
XMLFeedSpider example
@ -811,6 +812,6 @@ Combine SitemapSpider with other sources of urls::
.. _Sitemaps: https://www.sitemaps.org/index.html
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
.. _robots.txt: http://www.robotstxt.org/
.. _robots.txt: https://www.robotstxt.org/
.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

2
docs/topics/stats.rst
View File

@ -32,7 +32,7 @@ Common Stats Collector uses
Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
attribute. Here is an example of an extension that access stats::
class ExtensionThatAccessStats(object):
class ExtensionThatAccessStats:
def __init__(self, stats):
self.stats = stats

11
docs/topics/telnetconsole.rst
View File

@ -40,10 +40,10 @@ the console you need to type::
Connected to localhost.
Escape character is '^]'.
Username:
Password:
Password:
>>>
By default Username is ``scrapy`` and Password is autogenerated. The
By default Username is ``scrapy`` and Password is autogenerated. The
autogenerated Password can be seen on Scrapy logs like the example below::
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
@ -63,7 +63,7 @@ Available variables in the telnet console
=========================================
The telnet console is like a regular Python shell running inside the Scrapy
process, so you can do anything from it including importing new modules, etc.
process, so you can do anything from it including importing new modules, etc.
However, the telnet console comes with some default variables defined for
convenience:
@ -89,13 +89,11 @@ convenience:
+----------------+-------------------------------------------------------------------+
| ``prefs`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
| ``p`` | a shortcut to the `pprint.pprint`_ function |
| ``p`` | a shortcut to the :func:`pprint.pprint` function |
+----------------+-------------------------------------------------------------------+
| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
.. _pprint.pprint: https://docs.python.org/library/pprint.html#pprint.pprint
Telnet console usage examples
=============================
@ -208,4 +206,3 @@ Default: ``None``
The password used for the telnet console, default behaviour is to have it
autogenerated

85
docs/utils/linkfix.py
View File

@ -14,50 +14,57 @@ Author: dufferzafar
import re
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
def main():
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
# Used for remembering the file (and its contents)
# so we don't have to open the same file again.
_filename = None
_contents = None
# For every line, fix the respective file
for line in output_lines:
match = re.match(line_re, line)
# A regex that matches standard linkcheck output lines
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
if match:
newfilename = match.group(1)
errortype = match.group(2)
# Read lines from the linkcheck output file
try:
with open("build/linkcheck/output.txt") as out:
output_lines = out.readlines()
except IOError:
print("linkcheck output not found; please run linkcheck first.")
exit(1)
# Broken links can't be fixed and
# I am not sure what do with the local ones.
if errortype.lower() in ["broken", "local"]:
print("Not Fixed: " + line)
# For every line, fix the respective file
for line in output_lines:
match = re.match(line_re, line)
if match:
newfilename = match.group(1)
errortype = match.group(2)
# Broken links can't be fixed and
# I am not sure what do with the local ones.
if errortype.lower() in ["broken", "local"]:
print("Not Fixed: " + line)
else:
# If this is a new file
if newfilename != _filename:
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = _contents.replace(match.group(3), match.group(4))
else:
# If this is a new file
if newfilename != _filename:
# We don't understand what the current line means!
print("Not Understood: " + line)
# Update the previous file
if _filename:
with open(_filename, "w") as _file:
_file.write(_contents)
_filename = newfilename
# Read the new file to memory
with open(_filename) as _file:
_contents = _file.read()
_contents = _contents.replace(match.group(3), match.group(4))
else:
# We don't understand what the current line means!
print("Not Understood: " + line)
if __name__ == '__main__':
main()

83
extras/scrapy_zsh_completion
View File

@ -1,11 +1,12 @@
#compdef scrapy
_scrapy() {
local context state state_descr line
local ret=1
typeset -A opt_args
_arguments \
"(- 1 *)--help[Help]" \
"(- 1 *)"{-h,--help}"[Help]" \
"1: :->command" \
"*:: :->args"
"*:: :->args" && ret=0
case $state in
command)
@ -13,40 +14,40 @@ _scrapy() {
;;
args)
case $words[1] in
bench)
(bench)
_scrapy_glb_opts
;;
fetch)
(fetch)
local options=(
'--headers[print response HTTP headers instead of body]'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
genspider)
(genspider)
local options=(
{-l,--list}'[List available templates]'
{-e,--edit}'[Edit spider after creating it]'
{'(--list)-l','(-l)--list'}'[List available templates]'
{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
'--force[If the spider already exists, overwrite it with the template]'
{-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
{-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
'1:name:(NAME)'
'2:domain:_httpie_urls'
)
_scrapy_glb_opts $options
;;
runspider)
(runspider)
local options=(
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider file:_files -g \*.py'
)
_scrapy_glb_opts $options
;;
settings)
(settings)
local options=(
'--get=[print raw setting value]:option:(SETTING)'
'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
@ -56,77 +57,77 @@ _scrapy() {
)
_scrapy_glb_opts $options
;;
shell)
(shell)
local options=(
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'::file:_files -g \*.html'
'::URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
startproject)
(startproject)
local options=(
'1:name:(NAME)'
'2:dir:_dir_list'
)
_scrapy_glb_opts $options
;;
version)
(version)
local options=(
{-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
)
_scrapy_glb_opts $options
;;
view)
(view)
local options=(
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
'--spider[use this spider]:spider:_scrapy_spiders'
'--spider=[use this spider]:spider:_scrapy_spiders'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
;;
check)
(check)
local options=(
'(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
{-v,--verbose}'[print contract tests for all spiders]'
{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
crawl)
(crawl)
local options=(
{-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
edit)
(edit)
local options=(
'1:spider:_scrapy_spiders'
'1:spider:_scrapy_spiders'
)
_scrapy_glb_opts $options
;;
list)
(list)
_scrapy_glb_opts
;;
parse)
(parse)
local options=(
'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
'--spider[use this spider without looking for one]:spider:_scrapy_spiders'
'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
'--pipelines[process items through pipelines]'
"--nolinks[don't show links to follow (extracted requests)]"
"--noitems[don't show scraped items]"
'--nocolour[avoid using pygments to colorize the output]'
{-r,--rules}'[use CrawlSpider rules to discover the callback]'
{-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
{-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
{-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
{-v,--verbose}'[print each depth level one by one]'
{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
'1:URL:_httpie_urls'
)
_scrapy_glb_opts $options
@ -134,6 +135,8 @@ _scrapy() {
esac
;;
esac
return ret
}
_scrapy_cmds() {
@ -159,7 +162,7 @@ _scrapy_cmds() {
if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
commands=(${commands[@]} ${project_commands[@]})
fi
_describe -t common-commands 'common commands' commands
_describe -t common-commands 'common commands' commands && ret=0
}
_scrapy_glb_opts() {
@ -169,13 +172,13 @@ _scrapy_glb_opts() {
'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
'--pidfile=[write process ID to FILE]:file:_files'
'--profile=[write python cProfile stats to FILE]:file:_files'
'(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
'(-L --loglevel --logfile)--nolog[disable logging completely]'
'--pdb[enable pdb on failure]'
'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
)
options=(${options[@]} "$@")
_arguments $options
_arguments -A "-*" $options && ret=0
}
_httpie_urls() {

113
pylintrc Normal file
View File

@ -0,0 +1,113 @@
[MASTER]
persistent=no
jobs=1 # >1 hides results
[MESSAGES CONTROL]
disable=abstract-method,
anomalous-backslash-in-string,
arguments-differ,
attribute-defined-outside-init,
bad-classmethod-argument,
bad-continuation,
bad-indentation,
bad-mcs-classmethod-argument,
bad-super-call,
bad-whitespace,
bare-except,
blacklisted-name,
broad-except,
c-extension-no-member,
catching-non-exception,
cell-var-from-loop,
comparison-with-callable,
consider-iterating-dictionary,
consider-using-in,
consider-using-set-comprehension,
consider-using-sys-exit,
cyclic-import,
dangerous-default-value,
deprecated-method,
deprecated-module,
duplicate-code, # https://github.com/PyCQA/pylint/issues/214
eval-used,
expression-not-assigned,
fixme,
function-redefined,
global-statement,
import-error,
import-outside-toplevel,
import-self,
inconsistent-return-statements,
inherit-non-class,
invalid-name,
invalid-overridden-method,
isinstance-second-argument-not-valid-type,
keyword-arg-before-vararg,
line-too-long,
logging-format-interpolation,
logging-not-lazy,
lost-exception,
method-hidden,
misplaced-comparison-constant,
missing-docstring,
missing-final-newline,
multiple-imports,
multiple-statements,
no-else-continue,
no-else-raise,
no-else-return,
no-init,
no-member,
no-method-argument,
no-name-in-module,
no-self-argument,
no-self-use,
no-value-for-parameter,
not-an-iterable,
not-callable,
pointless-statement,
pointless-string-statement,
protected-access,
redefined-argument-from-local,
redefined-builtin,
redefined-outer-name,
reimported,
signature-differs,
singleton-comparison,
super-init-not-called,
superfluous-parens,
too-few-public-methods,
too-many-ancestors,
too-many-arguments,
too-many-branches,
too-many-format-args,
too-many-function-args,
too-many-instance-attributes,
too-many-lines,
too-many-locals,
too-many-public-methods,
too-many-return-statements,
trailing-newlines,
trailing-whitespace,
unbalanced-tuple-unpacking,
undefined-variable,
undefined-loop-variable,
unexpected-special-method-signature,
ungrouped-imports,
unidiomatic-typecheck,
unnecessary-comprehension,
unnecessary-lambda,
unnecessary-pass,
unreachable,
unsubscriptable-object,
unused-argument,
unused-import,
unused-variable,
unused-wildcard-import,
used-before-assignment,
useless-object-inheritance, # Required for Python 2 support
useless-return,
useless-super-delegation,
wildcard-import,
wrong-import-order,
wrong-import-position

240
pytest.ini
View File

@ -20,235 +20,23 @@ addopts =
twisted = 1
markers =
only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
flake8-max-line-length = 119
flake8-ignore =
# Files that are only meant to provide top-level imports are expected not
# to use any of their imports:
W503
# Exclude files that are meant to provide top-level imports
# E402: Module level import not at top of file
# F401: Module imported but unused
scrapy/__init__.py E402
scrapy/core/downloader/handlers/http.py F401
scrapy/http/__init__.py F401
scrapy/linkextractors/__init__.py E402 F401
scrapy/selector/__init__.py F401
scrapy/spiders/__init__.py E402 F401
# Issues pending a review:
# extras
extras/qps-bench-server.py E501
extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/parse.py E128 E501 E731 E226
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127 E306 E502
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E306 E128 W504
scrapy/core/spidermw.py E501 E731 E126 E226
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E305 E241
scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
scrapy/core/downloader/handlers/s3.py E501 E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
scrapy/extensions/closespider.py E501 E128 E123
scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
scrapy/extensions/httpcache.py E128 E501 E303
scrapy/extensions/memdebug.py E501
scrapy/extensions/spiderstate.py E501
scrapy/extensions/telnet.py E501 W504
scrapy/extensions/throttle.py E501
# scrapy/http
scrapy/http/common.py E501
scrapy/http/cookies.py E501
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E501 E402 W504
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
# scrapy/loader
scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/__init__.py E501
scrapy/pipelines/files.py E116 E501 E266
scrapy/pipelines/images.py E265 E501
scrapy/pipelines/media.py E125 E501 E266
# scrapy/selector
scrapy/selector/__init__.py F403
scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
scrapy/settings/default_settings.py E501 E114 E116 E226
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/asyncio.py E501
scrapy/utils/benchserver.py E501
scrapy/utils/conf.py E402 E501
scrapy/utils/console.py E306 E305
scrapy/utils/datatypes.py E501 E226
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/gz.py E305 E501 W504
scrapy/utils/http.py F403 E226
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
scrapy/utils/log.py E128 W503
scrapy/utils/http.py F403
scrapy/utils/markup.py F403
scrapy/utils/misc.py E501 E226
scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
scrapy/utils/reactor.py E226
scrapy/utils/reqser.py E501
scrapy/utils/request.py E127 E501
scrapy/utils/response.py E501 E128
scrapy/utils/signal.py E501 E128
scrapy/utils/sitemap.py E501
scrapy/utils/spider.py E271 E501
scrapy/utils/ssl.py E501
scrapy/utils/test.py E501
scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
scrapy/exporters.py E501 E226
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/resolver.py E501
scrapy/responsetypes.py E128 E501 E305
scrapy/robotstxt.py E501
scrapy/shell.py E501
scrapy/signalmanager.py E501
scrapy/spiderloader.py E225 F841 E501 E126
scrapy/squeues.py E128
scrapy/statscollectors.py E501
# tests
tests/__init__.py E402 E501
tests/mockserver.py E401 E501 E126 E123
tests/pipelines.py F841 E226
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/test_command_check.py E501
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128 E303 E226
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
tests/test_dependencies.py F841 E501 E305
tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501 E305
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E221 E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E306 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E301 E128 E265
tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
tests/test_logformatter.py E128 E501 E122
tests/test_mail.py E128 E501 E305
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
tests/test_pipeline_files.py E501 E303 E272 E226
tests/test_pipeline_images.py F841 E501 E303
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E305
tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_spider.py E501
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501 E226
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E303 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
tests/test_utils_defer.py E306 E501 F841 E226
tests/test_utils_deprecate.py F841 E306 E501
tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E303 E241
tests/test_utils_log.py E741 E226
tests/test_utils_python.py E501 E303 E731 E701 E305
tests/test_utils_reqser.py E501 E128
tests/test_utils_request.py E501 E128 E305
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731 E226
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_spider.py E305
tests/test_utils_template.py E305
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501
tests/test_utils_misc/__init__.py E501
scrapy/utils/url.py F403 F405
tests/test_loader.py E741

2
scrapy/VERSION
View File

@ -1 +1 @@
1.8.0
2.2.0

47
scrapy/__init__.py
View File

@ -2,33 +2,11 @@
Scrapy - a web crawling and web scraping framework written for Python
"""
__all__ = ['__version__', 'version_info', 'twisted_version',
'Spider', 'Request', 'FormRequest', 'Selector', 'Item', 'Field']
# Scrapy version
import pkgutil
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
version_info = tuple(int(v) if v.isdigit() else v
for v in __version__.split('.'))
del pkgutil
# Check minimum required Python version
import sys
if sys.version_info < (3, 5):
print("Scrapy %s requires Python 3.5" % __version__)
sys.exit(1)
# Ignore noisy twisted deprecation warnings
import warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Declare top-level shortcuts
from scrapy.spiders import Spider
@ -36,4 +14,29 @@ from scrapy.http import Request, FormRequest
from scrapy.selector import Selector
from scrapy.item import Item, Field
__all__ = [
'__version__', 'version_info', 'twisted_version', 'Spider',
'Request', 'FormRequest', 'Selector', 'Item', 'Field',
]
# Scrapy and Twisted versions
__version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split('.'))
twisted_version = (_txv.major, _txv.minor, _txv.micro)
# Check minimum required Python version
if sys.version_info < (3, 5, 2):
print("Scrapy %s requires Python 3.5.2" % __version__)
sys.exit(1)
# Ignore noisy twisted deprecation warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del pkgutil
del sys
del warnings

11
scrapy/_monkeypatches.py
View File

@ -1,11 +0,0 @@
import copyreg
# Undo what Twisted's perspective broker adds to pickle register
# to prevent bugs like Twisted#7989 while serializing requests
import twisted.persisted.styles # NOQA
# Remove only entries with twisted serializers for non-twisted types.
for k, v in frozenset(copyreg.dispatch_table.items()):
if not str(getattr(k, '__module__', '')).startswith('twisted') \
and str(getattr(v, '__module__', '')).startswith('twisted'):
copyreg.dispatch_table.pop(k)

7
scrapy/cmdline.py
View File

@ -12,7 +12,6 @@ from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
from scrapy.settings.deprecated import check_deprecated_settings
def _iter_command_classes(module_name):
@ -118,7 +117,6 @@ def execute(argv=None, settings=None):
pass
else:
settings['EDITOR'] = editor
check_deprecated_settings(settings)
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
@ -167,6 +165,7 @@ if __name__ == '__main__':
try:
execute()
finally:
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
# on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
# http://doc.pypy.org/en/latest/cpython_differences.html
# ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
garbage_collect()

43
scrapy/commands/__init__.py
View File

@ -5,11 +5,11 @@ import os
from optparse import OptionGroup
from twisted.python import failure
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.exceptions import UsageError
class ScrapyCommand(object):
class ScrapyCommand:
requires_project = False
crawler_process = None
@ -23,7 +23,8 @@ class ScrapyCommand(object):
self.settings = None # set in scrapy.cmdline
def set_crawler(self, crawler):
assert not hasattr(self, '_crawler'), "crawler already set"
if hasattr(self, '_crawler'):
raise RuntimeError("crawler already set")
self._crawler = crawler
def syntax(self):
@ -58,17 +59,17 @@ class ScrapyCommand(object):
"""
group = OptionGroup(parser, "Global Options")
group.add_option("--logfile", metavar="FILE",
help="log file. if omitted stderr will be used")
help="log file. if omitted stderr will be used")
group.add_option("-L", "--loglevel", metavar="LEVEL", default=None,
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
help="log level (default: %s)" % self.settings['LOG_LEVEL'])
group.add_option("--nolog", action="store_true",
help="disable logging completely")
help="disable logging completely")
group.add_option("--profile", metavar="FILE", default=None,
help="write python cProfile stats to FILE")
help="write python cProfile stats to FILE")
group.add_option("--pidfile", metavar="FILE",
help="write process ID to FILE")
help="write process ID to FILE")
group.add_option("-s", "--set", action="append", default=[], metavar="NAME=VALUE",
help="set/override setting (may be repeated)")
help="set/override setting (may be repeated)")
group.add_option("--pdb", action="store_true", help="enable pdb on failure")
parser.add_option_group(group)
@ -103,3 +104,27 @@ class ScrapyCommand(object):
Entry point for running commands
"""
raise NotImplementedError
class BaseRunSpiderCommand(ScrapyCommand):
"""
Common class used to share functionality between the crawl and runspider commands
"""
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE", action="append",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
self.settings.set('FEEDS', feeds, priority='cmdline')

2
scrapy/commands/bench.py
View File

@ -25,7 +25,7 @@ class Command(ScrapyCommand):
self.crawler_process.start()
class _BenchServer(object):
class _BenchServer:
def __enter__(self):
from scrapy.utils.test import get_testenv

50
scrapy/commands/crawl.py
View File

@ -1,11 +1,8 @@
import os
from scrapy.commands import ScrapyCommand
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.python import without_none_values
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = True
@ -15,38 +12,6 @@ class Command(ScrapyCommand):
def short_desc(self):
return "Run a spider"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
if opts.output == '-':
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(
self.settings.getwithbase('FEED_EXPORTERS'))
valid_output_formats = feed_exporters.keys()
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
if opts.output_format not in valid_output_formats:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
tuple(valid_output_formats)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
if len(args) < 1:
raise UsageError()
@ -54,8 +19,13 @@ class Command(ScrapyCommand):
raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
spname = args[0]
self.crawler_process.crawl(spname, **opts.spargs)
self.crawler_process.start()
crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
if self.crawler_process.bootstrap_failed:
if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
self.exitcode = 1
else:
self.crawler_process.start()
if self.crawler_process.bootstrap_failed or \
(hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
self.exitcode = 1

8
scrapy/commands/fetch.py
View File

@ -27,8 +27,8 @@ class Command(ScrapyCommand):
parser.add_option("--spider", dest="spider", help="use this spider")
parser.add_option("--headers", dest="headers", action="store_true",
help="print response HTTP headers instead of body")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():
@ -49,8 +49,8 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
cb = lambda x: self._print_response(x, opts)
request = Request(args[0], callback=cb, dont_filter=True)
request = Request(args[0], callback=self._print_response,
cb_kwargs={"opts": opts}, dont_filter=True)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:

17
scrapy/commands/genspider.py
View File

@ -36,15 +36,15 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-l", "--list", dest="list", action="store_true",
help="List available templates")
help="List available templates")
parser.add_option("-e", "--edit", dest="edit", action="store_true",
help="Edit spider after creating it")
help="Edit spider after creating it")
parser.add_option("-d", "--dump", dest="dump", metavar="TEMPLATE",
help="Dump template to standard output")
help="Dump template to standard output")
parser.add_option("-t", "--template", dest="template", default="basic",
help="Uses a custom template.")
help="Uses a custom template.")
parser.add_option("--force", dest="force", action="store_true",
help="If the spider already exists, overwrite it with the template")
help="If the spider already exists, overwrite it with the template")
def run(self, args, opts):
if opts.list:
@ -90,8 +90,7 @@ class Command(ScrapyCommand):
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() \
for s in module.split('_'))
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -102,8 +101,8 @@ class Command(ScrapyCommand):
spider_file = "%s.py" % join(spiders_dir, module)
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r " % (name, \
template_name), end=('' if spiders_module else '\n'))
print("Created spider %r using template %r "
% (name, template_name), end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))

41
scrapy/commands/parse.py
View File

@ -1,11 +1,11 @@
import json
import logging
from itemadapter import is_item, ItemAdapter
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils import display
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
@ -33,29 +33,29 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--spider", dest="spider", default=None,
help="use this spider without looking for one")
help="use this spider without looking for one")
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
help="set spider argument (may be repeated)")
parser.add_option("--pipelines", action="store_true",
help="process items through pipelines")
help="process items through pipelines")
parser.add_option("--nolinks", dest="nolinks", action="store_true",
help="don't show links to follow (extracted requests)")
help="don't show links to follow (extracted requests)")
parser.add_option("--noitems", dest="noitems", action="store_true",
help="don't show scraped items")
help="don't show scraped items")
parser.add_option("--nocolour", dest="nocolour", action="store_true",
help="avoid using pygments to colorize the output")
help="avoid using pygments to colorize the output")
parser.add_option("-r", "--rules", dest="rules", action="store_true",
help="use CrawlSpider rules to discover the callback")
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("--cbkwargs", dest="cbkwargs",
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
help="print each depth level one by one")
@property
def max_level(self):
@ -80,8 +80,8 @@ class Command(ScrapyCommand):
else:
items = self.items.get(lvl, [])
print("# Scraped Items ", "-"*60)
display.pprint([dict(x) for x in items], colorize=colour)
print("# Scraped Items ", "-" * 60)
display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
def print_requests(self, lvl=None, colour=True):
if lvl is None:
@ -92,14 +92,14 @@ class Command(ScrapyCommand):
else:
requests = self.requests.get(lvl, [])
print("# Requests ", "-"*65)
print("# Requests ", "-" * 65)
display.pprint(requests, colorize=colour)
def print_results(self, opts):
colour = not opts.nocolour
if opts.verbose:
for level in range(1, self.max_level+1):
for level in range(1, self.max_level + 1):
print('\n>>> DEPTH LEVEL: %s <<<' % level)
if not opts.noitems:
self.print_items(level, colour)
@ -117,7 +117,7 @@ class Command(ScrapyCommand):
items, requests = [], []
for x in iterate_spider_output(callback(response, **cb_kwargs)):
if isinstance(x, (BaseItem, dict)):
if is_item(x):
items.append(x)
elif isinstance(x, Request):
requests.append(x)
@ -146,9 +146,8 @@ class Command(ScrapyCommand):
if not self.spidercls:
logger.error('Unable to find spider for: %(url)s', {'url': url})
# Request requires callback argument as callable or None, not string
request = Request(url, None)
_start_requests = lambda s: [self.prepare_request(s, request, opts)]
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):

36
scrapy/commands/runspider.py
View File

@ -3,10 +3,8 @@ import os
from importlib import import_module
from scrapy.utils.spider import iter_spider_classes
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.conf import arglist_to_dict
from scrapy.utils.python import without_none_values
from scrapy.commands import BaseRunSpiderCommand
def _import_file(filepath):
@ -25,7 +23,7 @@ def _import_file(filepath):
return module
class Command(ScrapyCommand):
class Command(BaseRunSpiderCommand):
requires_project = False
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
@ -39,36 +37,6 @@ class Command(ScrapyCommand):
def long_desc(self):
return "Run the spider defined in the given file"
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
help="set spider argument (may be repeated)")
parser.add_option("-o", "--output", metavar="FILE",
help="dump scraped items into FILE (use - for stdout)")
parser.add_option("-t", "--output-format", metavar="FORMAT",
help="format to use for dumping items with -o")
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
if opts.output:
if opts.output == '-':
self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
if opts.output_format not in feed_exporters:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
tuple(feed_exporters)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
if len(args) != 1:
raise UsageError()

10
scrapy/commands/settings.py
View File

@ -19,15 +19,15 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--get", dest="get", metavar="SETTING",
help="print raw setting value")
help="print raw setting value")
parser.add_option("--getbool", dest="getbool", metavar="SETTING",
help="print setting value, interpreted as a boolean")
help="print setting value, interpreted as a boolean")
parser.add_option("--getint", dest="getint", metavar="SETTING",
help="print setting value, interpreted as an integer")
help="print setting value, interpreted as an integer")
parser.add_option("--getfloat", dest="getfloat", metavar="SETTING",
help="print setting value, interpreted as a float")
help="print setting value, interpreted as a float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING",
help="print setting value, interpreted as a list")
help="print setting value, interpreted as a list")
def run(self, args, opts):
settings = self.crawler_process.settings

8
scrapy/commands/shell.py
View File

@ -34,11 +34,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("-c", dest="code",
help="evaluate the code in the shell, print the result and exit")
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", default=False,
help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):
"""You can use this function to update the Scrapy objects that will be

30
scrapy/commands/startproject.py
View File

@ -1,5 +1,6 @@
import re
import os
import stat
import string
from importlib import import_module
from os.path import join, exists, abspath
@ -78,6 +79,29 @@ class Command(ScrapyCommand):
else:
copy2(srcname, dstname)
copystat(src, dst)
self._set_rw_permissions(dst)
def _set_rw_permissions(self, path):
"""
Sets permissions of a directory tree to +rw and +rwx for folders.
This is necessary if the start template files come without write
permissions.
"""
mode_rw = (stat.S_IRUSR
| stat.S_IWUSR
| stat.S_IRGRP
| stat.S_IROTH)
mode_x = (stat.S_IXUSR
| stat.S_IXGRP
| stat.S_IXOTH)
os.chmod(path, mode_rw | mode_x)
for root, dirs, files in os.walk(path):
for dir in dirs:
os.chmod(join(root, dir), mode_rw | mode_x)
for file in files:
os.chmod(join(root, file), mode_rw)
def run(self, args, opts):
if len(args) not in (1, 2):
@ -102,10 +126,8 @@ class Command(ScrapyCommand):
move(join(project_dir, 'module'), join(project_dir, project_name))
for paths in TEMPLATES_TO_RENDER:
path = join(*paths)
tplfile = join(project_dir,
string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name,
ProjectName=string_camelcase(project_name))
tplfile = join(project_dir, string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name, ProjectName=string_camelcase(project_name))
print("New Scrapy project '%s', using template directory '%s', "
"created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))

2
scrapy/commands/version.py
View File

@ -17,7 +17,7 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
parser.add_option("--verbose", "-v", dest="verbose", action="store_true",
help="also display twisted/python/platform info (useful for bug reports)")
help="also display twisted/python/platform info (useful for bug reports)")
def run(self, args, opts):
if opts.verbose:

8
scrapy/contracts/__init__.py
View File

@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
class ContractsManager(object):
class ContractsManager:
contracts = {}
def __init__(self, contracts):
@ -17,10 +17,10 @@ class ContractsManager(object):
self.contracts[contract.name] = contract
def tested_methods_from_spidercls(self, spidercls):
is_method = re.compile(r"^\s*@", re.MULTILINE).search
methods = []
for key, value in getmembers(spidercls):
if (callable(value) and value.__doc__ and
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
if callable(value) and value.__doc__ and is_method(value.__doc__):
methods.append(key)
return methods
@ -107,7 +107,7 @@ class ContractsManager(object):
request.errback = eb_wrapper
class Contract(object):
class Contract:
""" Abstract class for contracts """
request_cls = None

34
scrapy/contracts/default.py
View File

@ -1,10 +1,10 @@
import json
from scrapy.item import BaseItem
from scrapy.http import Request
from scrapy.exceptions import ContractFail
from itemadapter import is_item, ItemAdapter
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
from scrapy.http import Request
# contracts
@ -48,19 +48,23 @@ class ReturnsContract(Contract):
"""
name = 'returns'
objects = {
'request': Request,
'requests': Request,
'item': (BaseItem, dict),
'items': (BaseItem, dict),
object_type_verifiers = {
'request': lambda x: isinstance(x, Request),
'requests': lambda x: isinstance(x, Request),
'item': is_item,
'items': is_item,
}
def __init__(self, *args, **kwargs):
super(ReturnsContract, self).__init__(*args, **kwargs)
assert len(self.args) in [1, 2, 3]
if len(self.args) not in [1, 2, 3]:
raise ValueError(
"Incorrect argument quantity: expected 1, 2 or 3, got %i"
% len(self.args)
)
self.obj_name = self.args[0] or None
self.obj_type = self.objects[self.obj_name]
self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
try:
self.min_bound = int(self.args[1])
@ -75,7 +79,7 @@ class ReturnsContract(Contract):
def post_process(self, output):
occurrences = 0
for x in output:
if isinstance(x, self.obj_type):
if self.obj_type_verifier(x):
occurrences += 1
assertion = (self.min_bound <= occurrences <= self.max_bound)
@ -99,8 +103,8 @@ class ScrapesContract(Contract):
def post_process(self, output):
for x in output:
if isinstance(x, (BaseItem, dict)):
missing = [arg for arg in self.args if arg not in x]
if is_item(x):
missing = [arg for arg in self.args if arg not in ItemAdapter(x)]
if missing:
raise ContractFail(
"Missing fields: %s" % ", ".join(missing))
missing_str = ", ".join(missing)
raise ContractFail("Missing fields: %s" % missing_str)

12
scrapy/core/downloader/__init__.py
View File

@ -3,7 +3,7 @@ from time import time
from datetime import datetime
from collections import deque
from twisted.internet import reactor, defer, task
from twisted.internet import defer, task
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
@ -13,7 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import DownloadHandlers
class Slot(object):
class Slot:
"""Downloader slot"""
def __init__(self, concurrency, delay, randomize_delay):
@ -66,7 +66,7 @@ def _get_concurrency_delay(concurrency, spider, settings):
return concurrency, delay
class Downloader(object):
class Downloader:
DOWNLOAD_SLOT = 'download_slot'
@ -133,6 +133,7 @@ class Downloader(object):
return deferred
def _process_queue(self, spider, slot):
from twisted.internet import reactor
if slot.latercall and slot.latercall.active():
return
@ -172,7 +173,7 @@ class Downloader(object):
return response
dfd.addCallback(_downloaded)
# 3. After response arrives, remove the request from transferring
# 3. After response arrives, remove the request from transferring
# state to free up the transferring slot so it can be used by the
# following requests (perhaps those which came from the downloader
# middleware itself)
@ -181,6 +182,9 @@ class Downloader(object):
def finish_transferring(_):
slot.transferring.remove(request)
self._process_queue(spider, slot)
self.signals.send_catch_log(signal=signals.request_left_downloader,
request=request,
spider=spider)
return _
return dfd.addBoth(finish_transferring)

21
scrapy/core/downloader/contextfactory.py
View File

@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
#
# * getattr() for `_ssl_method` attribute for context factories
# not calling super(..., self).__init__
return CertificateOptions(verify=False,
method=getattr(self, 'method',
getattr(self, '_ssl_method', None)),
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers)
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers,
)
# kept for old-style HTTP/1.0 downloader context twisted calls,
# e.g. connectSSL()
@ -86,8 +87,8 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
#
# This means that a website like https://www.cacert.org will be rejected
# by default, since CAcert.org CA certificate is seldom shipped.
return optionsForClientTLS(hostname.decode("ascii"),
trustRoot=platformTrust(),
extraCertificateOptions={
'method': self._ssl_method,
})
return optionsForClientTLS(
hostname=hostname.decode("ascii"),
trustRoot=platformTrust(),
extraCertificateOptions={'method': self._ssl_method},
)

20
scrapy/core/downloader/handlers/ftp.py
View File

@ -32,7 +32,6 @@ import re
from io import BytesIO
from urllib.parse import unquote
from twisted.internet import reactor
from twisted.internet.protocol import ClientCreator, Protocol
from twisted.protocols.ftp import CommandFailed, FTPClient
@ -81,24 +80,25 @@ class FTPDownloadHandler:
return cls(crawler.settings)
def download_request(self, request, spider):
from twisted.internet import reactor
parsed_url = urlparse_cached(request)
user = request.meta.get("ftp_user", self.default_user)
password = request.meta.get("ftp_password", self.default_password)
passive_mode = 1 if bool(request.meta.get("ftp_passive",
self.passive_mode)) else 0
creator = ClientCreator(reactor, FTPClient, user, password,
passive=passive_mode)
return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
request, unquote(parsed_url.path))
creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode)
dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
def gotClient(self, client, request, filepath):
self.client = client
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
return client.retrieveFile(filepath, protocol)\
.addCallbacks(callback=self._build_response,
callbackArgs=(request, protocol),
errback=self._failed,
errbackArgs=(request,))
return client.retrieveFile(filepath, protocol).addCallbacks(
callback=self._build_response,
callbackArgs=(request, protocol),
errback=self._failed,
errbackArgs=(request,),
)
def _build_response(self, result, request, protocol):
self.result = result

3
scrapy/core/downloader/handlers/http10.py
View File

@ -1,7 +1,5 @@
"""Download handlers for http and https schemes
"""
from twisted.internet import reactor
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_unicode
@ -26,6 +24,7 @@ class HTTP10DownloadHandler:
return factory.deferred
def _connect(self, factory):
from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
client_context_factory = create_instance(

118
scrapy/core/downloader/handlers/http11.py
View File

@ -1,24 +1,28 @@
"""Download handlers for http and https schemes"""
import ipaddress
import logging
import re
import warnings
from contextlib import suppress
from io import BytesIO
from time import time
from urllib.parse import urldefrag
from twisted.internet import defer, protocol, reactor
from twisted.internet import defer, protocol, ssl
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.python.failure import Failure
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
from zope.interface import implementer
from scrapy import signals
from scrapy.core.downloader.tls import openssl_methods
from scrapy.core.downloader.webclient import _parse
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.exceptions import ScrapyDeprecationWarning, StopDownload
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.utils.misc import create_instance, load_object
@ -32,6 +36,9 @@ class HTTP11DownloadHandler:
lazy = False
def __init__(self, settings, crawler=None):
self._crawler = crawler
from twisted.internet import reactor
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
@ -76,10 +83,12 @@ class HTTP11DownloadHandler:
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
fail_on_dataloss=self._fail_on_dataloss,
crawler=self._crawler,
)
return agent.download_request(request)
def close(self):
from twisted.internet import reactor
d = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
@ -265,14 +274,14 @@ class ScrapyProxyAgent(Agent):
)
class ScrapyAgent(object):
class ScrapyAgent:
_Agent = Agent
_ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
maxsize=0, warnsize=0, fail_on_dataloss=True):
maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):
self._contextFactory = contextFactory
self._connectTimeout = connectTimeout
self._bindAddress = bindAddress
@ -281,8 +290,10 @@ class ScrapyAgent(object):
self._warnsize = warnsize
self._fail_on_dataloss = fail_on_dataloss
self._txresponse = None
self._crawler = crawler
def _get_agent(self, request, timeout):
from twisted.internet import reactor
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
if proxy:
@ -325,6 +336,7 @@ class ScrapyAgent(object):
)
def download_request(self, request):
from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connectTimeout
agent = self._get_agent(request, timeout)
@ -336,20 +348,6 @@ class ScrapyAgent(object):
headers.removeHeader(b'Proxy-Authorization')
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
elif method == b'POST':
# Setting Content-Length: 0 even for POST requests is not a
# MUST per HTTP RFCs, but it's common behavior, and some
# servers require this, otherwise returning HTTP 411 Length required
#
# RFC 7230#section-3.3.2:
# "a Content-Length header field is normally sent in a POST
# request even when the value is 0 (indicating an empty payload body)."
#
# Twisted < 17 will not add "Content-Length: 0" by itself;
# Twisted >= 17 fixes this;
# Using a producer with an empty-string sends `0` as Content-Length
# for all versions of Twisted.
bodyproducer = _RequestBodyProducer(b'')
else:
bodyproducer = None
start_time = time()
@ -382,7 +380,13 @@ class ScrapyAgent(object):
def _cb_bodyready(self, txresponse, request):
# deliverBody hangs for responses without body
if txresponse.length == 0:
return txresponse, b'', None
return {
"txresponse": txresponse,
"body": b"",
"flags": None,
"certificate": None,
"ip_address": None,
}
maxsize = request.meta.get('download_maxsize', self._maxsize)
warnsize = request.meta.get('download_warnsize', self._warnsize)
@ -409,7 +413,15 @@ class ScrapyAgent(object):
d = defer.Deferred(_cancel)
txresponse.deliverBody(
_ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
_ResponseReader(
finished=d,
txresponse=txresponse,
request=request,
maxsize=maxsize,
warnsize=warnsize,
fail_on_dataloss=fail_on_dataloss,
crawler=self._crawler,
)
)
# save response for timeouts
@ -418,15 +430,25 @@ class ScrapyAgent(object):
return d
def _cb_bodydone(self, result, request, url):
txresponse, body, flags = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
response = respcls(
url=url,
status=int(result["txresponse"].code),
headers=headers,
body=result["body"],
flags=result["flags"],
certificate=result["certificate"],
ip_address=result["ip_address"],
)
if result.get("failure"):
result["failure"].value.response = response
return result["failure"]
return response
@implementer(IBodyProducer)
class _RequestBodyProducer(object):
class _RequestBodyProducer:
def __init__(self, body):
self.body = body
@ -445,7 +467,7 @@ class _RequestBodyProducer(object):
class _ResponseReader(protocol.Protocol):
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):
self._finished = finished
self._txresponse = txresponse
self._request = request
@ -456,6 +478,27 @@ class _ResponseReader(protocol.Protocol):
self._fail_on_dataloss_warned = False
self._reached_warnsize = False
self._bytes_received = 0
self._certificate = None
self._ip_address = None
self._crawler = crawler
def _finish_response(self, flags=None, failure=None):
self._finished.callback({
"txresponse": self._txresponse,
"body": self._bodybuf.getvalue(),
"flags": flags,
"certificate": self._certificate,
"ip_address": self._ip_address,
"failure": failure,
})
def connectionMade(self):
if self._certificate is None:
with suppress(AttributeError):
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
if self._ip_address is None:
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
def dataReceived(self, bodyBytes):
# This maybe called several times after cancel was called with buffered data.
@ -465,6 +508,20 @@ class _ResponseReader(protocol.Protocol):
self._bodybuf.write(bodyBytes)
self._bytes_received += len(bodyBytes)
bytes_received_result = self._crawler.signals.send_catch_log(
signal=signals.bytes_received,
data=bodyBytes,
request=self._request,
spider=self._crawler.spider,
)
for handler, result in bytes_received_result:
if isinstance(result, Failure) and isinstance(result.value, StopDownload):
logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
{"request": self._request, "handler": handler.__qualname__})
self.transport._producer.loseConnection()
failure = result if result.value.fail else None
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
@ -486,18 +543,17 @@ class _ResponseReader(protocol.Protocol):
if self._finished.called:
return
body = self._bodybuf.getvalue()
if reason.check(ResponseDone):
self._finished.callback((self._txresponse, body, None))
self._finish_response()
return
if reason.check(PotentialDataLoss):
self._finished.callback((self._txresponse, body, ['partial']))
self._finish_response(flags=["partial"])
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if not self._fail_on_dataloss:
self._finished.callback((self._txresponse, body, ['dataloss']))
self._finish_response(flags=["dataloss"])
return
elif not self._fail_on_dataloss_warned:

13
scrapy/core/downloader/handlers/s3.py
View File

@ -100,11 +100,12 @@ class S3DownloadHandler:
url=url, headers=awsrequest.headers.items())
else:
signed_headers = self.conn.make_request(
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body)
method=request.method,
bucket=bucket,
key=unquote(p.path),
query_args=unquote(p.query),
headers=request.headers,
data=request.body,
)
request = request.replace(url=url, headers=signed_headers)
return self._download_http(request, spider)

41
scrapy/core/downloader/middleware.py
View File

@ -35,38 +35,45 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, response.__class__.__name__))
raise _InvalidOutput(
"Middleware %s.process_request must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, response.__class__.__name__)
)
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
return response
return (yield download_func(request=request, spider=spider))
@defer.inlineCallbacks
def process_response(response):
assert response is not None, 'Received None in process_response'
if isinstance(response, Request):
defer.returnValue(response)
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
return response
for method in self.methods['process_response']:
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))
raise _InvalidOutput(
"Middleware %s.process_response must return Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
)
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
return response
return response
@defer.inlineCallbacks
def process_exception(_failure):
exception = _failure.value
def process_exception(failure):
exception = failure.value
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))
raise _InvalidOutput(
"Middleware %s.process_exception must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
)
if response:
defer.returnValue(response)
defer.returnValue(_failure)
return response
return failure
deferred = mustbe_deferred(process_request, request)
deferred.addErrback(process_exception)

5
scrapy/core/downloader/tls.py
View File

@ -20,8 +20,8 @@ METHOD_TLSv12 = 'TLSv1.2'
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
@ -89,4 +89,5 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
'from host "{}" (exception: {})'.format(
self._hostnameASCII, repr(e)))
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')

15
scrapy/core/downloader/webclient.py
View File

@ -14,13 +14,12 @@ from scrapy.responsetypes import responsetypes
def _parsed_url_args(parsed):
# Assume parsed is urlparse-d from Request.url,
# which was passed via safe_url_string and is ascii-only.
b = lambda s: to_bytes(s, encoding='ascii')
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
path = b(path)
host = b(parsed.hostname)
path = to_bytes(path, encoding="ascii")
host = to_bytes(parsed.hostname, encoding="ascii")
port = parsed.port
scheme = b(parsed.scheme)
netloc = b(parsed.netloc)
scheme = to_bytes(parsed.scheme, encoding="ascii")
netloc = to_bytes(parsed.netloc, encoding="ascii")
if port is None:
port = 443 if scheme == b'https' else 80
return scheme, netloc, host, port, path
@ -89,8 +88,8 @@ class ScrapyHTTPPageGetter(HTTPClient):
self.transport.stopProducing()
self.factory.noPage(
defer.TimeoutError("Getting %s took longer than %s seconds." %
(self.factory.url, self.factory.timeout)))
defer.TimeoutError("Getting %s took longer than %s seconds."
% (self.factory.url, self.factory.timeout)))
class ScrapyHTTPClientFactory(HTTPClientFactory):
@ -140,7 +139,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
self.headers['Content-Length'] = 0
def _build_response(self, body, request):
request.meta['download_latency'] = self.headers_time-self.start_time
request.meta['download_latency'] = self.headers_time - self.start_time
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)

50
scrapy/core/engine.py
View File

@ -21,7 +21,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
logger = logging.getLogger(__name__)
class Slot(object):
class Slot:
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
@ -53,7 +53,7 @@ class Slot(object):
self.closing.callback(None)
class ExecutionEngine(object):
class ExecutionEngine:
def __init__(self, crawler, spider_closed_callback):
self.crawler = crawler
@ -73,7 +73,8 @@ class ExecutionEngine(object):
@defer.inlineCallbacks
def start(self):
"""Start the execution engine"""
assert not self.running, "Engine already running"
if self.running:
raise RuntimeError("Engine already running")
self.start_time = time()
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
@ -82,7 +83,8 @@ class ExecutionEngine(object):
def stop(self):
"""Stop the execution engine gracefully"""
assert self.running, "Engine not running"
if not self.running:
raise RuntimeError("Engine not running")
self.running = False
dfd = self._close_all_spiders()
return dfd.addBoth(lambda _: self._finish_stopping_engine())
@ -165,7 +167,11 @@ class ExecutionEngine(object):
return d
def _handle_downloader_output(self, response, request, spider):
assert isinstance(response, (Request, Response, Failure)), response
if not isinstance(response, (Request, Response, Failure)):
raise TypeError(
"Incorrect type: expected Request, Response or Failure, got %s: %r"
% (type(response), response)
)
# downloader middleware can return requests (for example, redirects)
if isinstance(response, Request):
self.crawl(response, spider)
@ -205,17 +211,15 @@ class ExecutionEngine(object):
return not bool(self.slot)
def crawl(self, request, spider):
assert spider in self.open_spiders, \
"Spider %r not opened when crawling: %s" % (spider.name, request)
if spider not in self.open_spiders:
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
self.schedule(request, spider)
self.slot.nextcall.schedule()
def schedule(self, request, spider):
self.signals.send_catch_log(signal=signals.request_scheduled,
request=request, spider=spider)
self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider)
if not self.slot.scheduler.enqueue_request(request):
self.signals.send_catch_log(signal=signals.request_dropped,
request=request, spider=spider)
self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider)
def download(self, request, spider):
d = self._download(request, spider)
@ -224,21 +228,25 @@ class ExecutionEngine(object):
def _downloaded(self, response, slot, request, spider):
slot.remove_request(request)
return self.download(response, spider) \
if isinstance(response, Request) else response
return self.download(response, spider) if isinstance(response, Request) else response
def _download(self, request, spider):
slot = self.slot
slot.add_request(request)
def _on_success(response):
assert isinstance(response, (Response, Request))
if not isinstance(response, (Response, Request)):
raise TypeError(
"Incorrect type: expected Response or Request, got %s: %r"
% (type(response), response)
)
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
if logkws is not None:
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
self.signals.send_catch_log(signal=signals.response_received,
response=response, request=request, spider=spider)
self.signals.send_catch_log(signals.response_received,
response=response, request=request, spider=spider)
return response
def _on_complete(_):
@ -252,8 +260,8 @@ class ExecutionEngine(object):
@defer.inlineCallbacks
def open_spider(self, spider, start_requests=(), close_if_idle=True):
assert self.has_capacity(), "No free spider slot when opening %r" % \
spider.name
if not self.has_capacity():
raise RuntimeError("No free spider slot when opening %r" % spider.name)
logger.info("Spider opened", extra={'spider': spider})
nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_crawler(self.crawler)
@ -276,10 +284,8 @@ class ExecutionEngine(object):
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
res = self.signals.send_catch_log(signal=signals.spider_idle, \
spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
for _, x in res):
res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
return
if self.spider_is_idle(spider):

35
scrapy/core/scheduler.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning
logger = logging.getLogger(__name__)
class Scheduler(object):
class Scheduler:
"""
Scrapy Scheduler. It allows to enqueue requests and then get
a next request to download. Scheduler is also handling duplication
@ -66,8 +66,7 @@ class Scheduler(object):
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
settings.getbool('SCHEDULER_DEBUG'))
logunser = settings.getbool('SCHEDULER_DEBUG')
return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
mqclass=mqclass, crawler=crawler)
@ -119,7 +118,7 @@ class Scheduler(object):
if self.dqs is None:
return
try:
self.dqs.push(request, -request.priority)
self.dqs.push(request)
except ValueError as e: # non serializable request
if self.logunser:
msg = ("Unable to serialize request: %(request)s - reason:"
@ -135,35 +134,29 @@ class Scheduler(object):
return True
def _mqpush(self, request):
self.mqs.push(request, -request.priority)
self.mqs.push(request)
def _dqpop(self):
if self.dqs:
return self.dqs.pop()
def _newmq(self, priority):
""" Factory for creating memory queues. """
return self.mqclass()
def _newdq(self, priority):
""" Factory for creating disk queues. """
path = join(self.dqdir, 'p%s' % (priority, ))
return self.dqclass(path)
def _mq(self):
""" Create a new priority queue instance, with in-memory storage """
return create_instance(self.pqclass, None, self.crawler, self._newmq,
serialize=False)
return create_instance(self.pqclass,
settings=None,
crawler=self.crawler,
downstream_queue_cls=self.mqclass,
key='')
def _dq(self):
""" Create a new priority queue instance, with disk storage """
state = self._read_dqs_state(self.dqdir)
q = create_instance(self.pqclass,
None,
self.crawler,
self._newdq,
state,
serialize=True)
settings=None,
crawler=self.crawler,
downstream_queue_cls=self.dqclass,
key=self.dqdir,
startprios=state)
if q:
logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
{'queuesize': len(q)}, extra={'spider': self.spider})

65
scrapy/core/scraper.py
View File

@ -4,25 +4,24 @@ extracts information from them"""
import logging
from collections import deque
from twisted.python.failure import Failure
from itemadapter import is_item
from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy import signals
from scrapy.http import Request, Response
from scrapy.item import BaseItem
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.utils.request import referer_str
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.spider import iterate_spider_output
logger = logging.getLogger(__name__)
class Slot(object):
class Slot:
"""Scraper slot (one per running spider)"""
MIN_RESPONSE_SIZE = 1024
@ -63,7 +62,7 @@ class Slot(object):
return self.active_size > self.max_active_size
class Scraper(object):
class Scraper:
def __init__(self, crawler):
self.slot = None
@ -124,7 +123,11 @@ class Scraper(object):
def _scrape(self, response, request, spider):
"""Handle the downloaded response or failure through the spider
callback/errback"""
assert isinstance(response, (Response, Failure))
if not isinstance(response, (Response, Failure)):
raise TypeError(
"Incorrect type: expected Response or Failure, got %s: %r"
% (type(response), response)
)
dfd = self._scrape2(response, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
@ -158,9 +161,9 @@ class Scraper(object):
if isinstance(exc, CloseSpider):
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
return
logger.error(
"Spider error processing %(request)s (referer: %(referer)s)",
{'request': request, 'referer': referer_str(request)},
logkws = self.logformatter.spider_error(_failure, request, response, spider)
logger.log(
*logformatter_adapter(logkws),
exc_info=failure_to_exc_info(_failure),
extra={'spider': spider}
)
@ -188,7 +191,7 @@ class Scraper(object):
"""
if isinstance(output, Request):
self.crawler.engine.crawl(request=output, spider=spider)
elif isinstance(output, (BaseItem, dict)):
elif is_item(output):
self.slot.itemproc_size += 1
dfd = self.itemproc.process_item(output, spider)
dfd.addBoth(self._itemproc_finished, output, response, spider)
@ -197,10 +200,11 @@ class Scraper(object):
pass
else:
typename = type(output).__name__
logger.error('Spider must return Request, BaseItem, dict or None, '
'got %(typename)r in %(request)s',
{'request': request, 'typename': typename},
extra={'spider': spider})
logger.error(
'Spider must return request, item, or None, got %(typename)r in %(request)s',
{'request': request, 'typename': typename},
extra={'spider': spider},
)
def _log_download_errors(self, spider_failure, download_failure, request, spider):
"""Log and silence errors that come from the engine (typically download
@ -208,16 +212,21 @@ class Scraper(object):
"""
if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
if download_failure.frames:
logger.error('Error downloading %(request)s',
{'request': request},
exc_info=failure_to_exc_info(download_failure),
extra={'spider': spider})
logkws = self.logformatter.download_error(download_failure, request, spider)
logger.log(
*logformatter_adapter(logkws),
extra={'spider': spider},
exc_info=failure_to_exc_info(download_failure),
)
else:
errmsg = download_failure.getErrorMessage()
if errmsg:
logger.error('Error downloading %(request)s: %(errmsg)s',
{'request': request, 'errmsg': errmsg},
extra={'spider': spider})
logkws = self.logformatter.download_error(
download_failure, request, spider, errmsg)
logger.log(
*logformatter_adapter(logkws),
extra={'spider': spider},
)
if spider_failure is not download_failure:
return spider_failure
@ -236,7 +245,7 @@ class Scraper(object):
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
logkws = self.logformatter.error(item, ex, response, spider)
logkws = self.logformatter.item_error(item, ex, response, spider)
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(

70
scrapy/core/spidermw.py
View File

@ -3,13 +3,14 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
from itertools import chain, islice
from itertools import islice
from twisted.python.failure import Failure
from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.python import MutableChain
@ -17,6 +18,13 @@ def _isiterable(possible_iterator):
return hasattr(possible_iterator, '__iter__')
def _fname(f):
return "{}.{}".format(
f.__self__.__class__.__name__,
f.__func__.__name__
)
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
@ -31,27 +39,36 @@ class SpiderMiddlewareManager(MiddlewareManager):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].appendleft(mw.process_start_requests)
self.methods['process_spider_output'].appendleft(getattr(mw, 'process_spider_output', None))
self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
process_spider_output = getattr(mw, 'process_spider_output', None)
self.methods['process_spider_output'].appendleft(process_spider_output)
process_spider_exception = getattr(mw, 'process_spider_exception', None)
self.methods['process_spider_exception'].appendleft(process_spider_exception)
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f: '%s.%s' % (
f.__self__.__class__.__name__,
f.__func__.__name__)
def process_spider_input(response):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
if result is not None:
raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
.format(fname(method), type(result)))
msg = "Middleware {} must return None or raise an exception, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
except _InvalidOutput:
raise
except Exception:
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
def _evaluate_iterable(iterable, exception_processor_index, recover_to):
try:
for r in iterable:
yield r
except Exception as ex:
exception_result = process_spider_exception(Failure(ex), exception_processor_index)
if isinstance(exception_result, Failure):
raise
recover_to.extend(exception_result)
def process_spider_exception(_failure, start_index=0):
exception = _failure.value
# don't handle _InvalidOutput exception
@ -65,12 +82,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
return process_spider_output(result, method_index+1)
return process_spider_output(result, method_index + 1)
elif result is None:
continue
else:
raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
.format(fname(method), type(result)))
msg = "Middleware {} must return None or an iterable, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
return _failure
def process_spider_output(result, start_index=0):
@ -78,38 +95,33 @@ class SpiderMiddlewareManager(MiddlewareManager):
# chain, they went through it already from the process_spider_exception method
recovered = MutableChain()
def evaluate_iterable(iterable, index):
try:
for r in iterable:
yield r
except Exception as ex:
exception_result = process_spider_exception(Failure(ex), index+1)
if isinstance(exception_result, Failure):
raise
recovered.extend(exception_result)
method_list = islice(self.methods['process_spider_output'], start_index, None)
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
continue
# the following might fail directly if the output value is not a generator
try:
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=spider)
except Exception as ex:
exception_result = process_spider_exception(Failure(ex), method_index+1)
exception_result = process_spider_exception(Failure(ex), method_index + 1)
if isinstance(exception_result, Failure):
raise
return exception_result
if _isiterable(result):
result = evaluate_iterable(result, method_index)
result = _evaluate_iterable(result, method_index + 1, recovered)
else:
raise _InvalidOutput('Middleware {} must return an iterable, got {}'
.format(fname(method), type(result)))
msg = "Middleware {} must return an iterable, got {}"
raise _InvalidOutput(msg.format(_fname(method), type(result)))
return chain(result, recovered)
return MutableChain(result, recovered)
def process_callback_output(result):
recovered = MutableChain()
result = _evaluate_iterable(result, 0, recovered)
return MutableChain(process_spider_output(result), recovered)
dfd = mustbe_deferred(process_spider_input, response)
dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception)
dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
return dfd
def process_start_requests(self, start_requests, spider):

78
scrapy/crawler.py
View File

@ -4,7 +4,15 @@ import signal
import warnings
from twisted.internet import defer
from zope.interface.verify import DoesNotImplement, verifyClass
from zope.interface.exceptions import DoesNotImplement
try:
# zope >= 5.0 only supports MultipleInvalid
from zope.interface.exceptions import MultipleInvalid
except ImportError:
MultipleInvalid = None
from zope.interface.verify import verifyClass
from scrapy import signals, Spider
from scrapy.core.engine import ExecutionEngine
@ -13,7 +21,6 @@ from scrapy.extension import ExtensionManager
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import overridden_settings, Settings
from scrapy.signalmanager import SignalManager
from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
from scrapy.utils.log import (
configure_logging,
get_scrapy_root_handler,
@ -23,6 +30,7 @@ from scrapy.utils.log import (
)
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.reactor import install_reactor, verify_installed_reactor
logger = logging.getLogger(__name__)
@ -68,20 +76,10 @@ class Crawler:
self.spider = None
self.engine = None
@property
def spiders(self):
if not hasattr(self, '_spiders'):
warnings.warn("Crawler.spiders is deprecated, use "
"CrawlerRunner.spider_loader or instantiate "
"scrapy.spiderloader.SpiderLoader with your "
"settings.",
category=ScrapyDeprecationWarning, stacklevel=2)
self._spiders = _get_spider_loader(self.settings.frozencopy())
return self._spiders
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
assert not self.crawling, "Crawling already taking place"
if self.crawling:
raise RuntimeError("Crawling already taking place")
self.crawling = True
try:
@ -130,15 +128,32 @@ class CrawlerRunner:
":meth:`crawl` and managed by this class."
)
@staticmethod
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
cls_path = settings.get('SPIDER_LOADER_CLASS')
loader_cls = load_object(cls_path)
excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement
try:
verifyClass(ISpiderLoader, loader_cls)
except excs:
warnings.warn(
'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
'Please add all missing methods to avoid unexpected runtime errors.',
category=ScrapyDeprecationWarning, stacklevel=2
)
return loader_cls.from_settings(settings.frozencopy())
def __init__(self, settings=None):
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
self.settings = settings
self.spider_loader = _get_spider_loader(settings)
self.spider_loader = self._get_spider_loader(settings)
self._crawlers = set()
self._active = set()
self.bootstrap_failed = False
self._handle_asyncio_reactor()
self._handle_twisted_reactor()
@property
def spiders(self):
@ -232,10 +247,9 @@ class CrawlerRunner:
while self._active:
yield defer.DeferredList(self._active)
def _handle_asyncio_reactor(self):
if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():
raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "
"reactor is not installed.")
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
verify_installed_reactor(self.settings["TWISTED_REACTOR"])
class CrawlerProcess(CrawlerRunner):
@ -324,23 +338,7 @@ class CrawlerProcess(CrawlerRunner):
except RuntimeError: # raised if already stopped or in shutdown stage
pass
def _handle_asyncio_reactor(self):
if self.settings.getbool('ASYNCIO_REACTOR'):
install_asyncio_reactor()
super()._handle_asyncio_reactor()
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
cls_path = settings.get('SPIDER_LOADER_CLASS')
loader_cls = load_object(cls_path)
try:
verifyClass(ISpiderLoader, loader_cls)
except DoesNotImplement:
warnings.warn(
'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does '
'not fully implement scrapy.interfaces.ISpiderLoader interface. '
'Please add all missing methods to avoid unexpected runtime errors.',
category=ScrapyDeprecationWarning, stacklevel=2
)
return loader_cls.from_settings(settings.frozencopy())
def _handle_twisted_reactor(self):
if self.settings.get("TWISTED_REACTOR"):
install_reactor(self.settings["TWISTED_REACTOR"])
super()._handle_twisted_reactor()

5
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
import re
import logging
@ -11,7 +10,7 @@ from scrapy.http import HtmlResponse
logger = logging.getLogger(__name__)
class AjaxCrawlMiddleware(object):
class AjaxCrawlMiddleware:
"""
Handle 'AJAX crawlable' pages marked as crawlable via meta tag.
For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
@ -47,7 +46,7 @@ class AjaxCrawlMiddleware(object):
return response
# scrapy already handles #! links properly
ajax_crawl_request = request.replace(url=request.url+'#!')
ajax_crawl_request = request.replace(url=request.url + '#!')
logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
{'ajax_crawl_request': ajax_crawl_request, 'request': request},
extra={'spider': spider})

21
scrapy/downloadermiddlewares/chunked.py
View File

@ -1,21 +0,0 @@
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.http import decode_chunked_transfer
warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
"chunked transfers are supported by default.",
ScrapyDeprecationWarning, stacklevel=2)
class ChunkedTransferMiddleware(object):
"""This middleware adds support for chunked transfer encoding, as
documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
"""
def process_response(self, request, response, spider):
if response.headers.get('Transfer-Encoding') == 'chunked':
body = decode_chunked_transfer(response.body)
return response.replace(body=body)
return response

80
scrapy/downloadermiddlewares/cookies.py
View File

@ -10,7 +10,7 @@ from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
class CookiesMiddleware(object):
class CookiesMiddleware:
"""This middleware enables working with sites that need cookies"""
def __init__(self, debug=False):
@ -29,8 +29,7 @@ class CookiesMiddleware(object):
cookiejarkey = request.meta.get("cookiejar")
jar = self.jars[cookiejarkey]
cookies = self._get_request_cookies(jar, request)
for cookie in cookies:
for cookie in self._get_request_cookies(jar, request):
jar.set_cookie_if_ok(cookie, request)
# set Cookie header
@ -68,28 +67,65 @@ class CookiesMiddleware(object):
msg = "Received cookies from: {}\n{}".format(response, cookies)
logger.debug(msg, extra={'spider': spider})
def _format_cookie(self, cookie):
# build cookie string
cookie_str = '%s=%s' % (cookie['name'], cookie['value'])
if cookie.get('path', None):
cookie_str += '; Path=%s' % cookie['path']
if cookie.get('domain', None):
cookie_str += '; Domain=%s' % cookie['domain']
def _format_cookie(self, cookie, request):
"""
Given a dict consisting of cookie components, return its string representation.
Decode from bytes if necessary.
"""
decoded = {}
for key in ("name", "value", "path", "domain"):
if not cookie.get(key):
if key in ("name", "value"):
msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
logger.warning(msg.format(request, cookie, key))
return
continue
if isinstance(cookie[key], str):
decoded[key] = cookie[key]
else:
try:
decoded[key] = cookie[key].decode("utf8")
except UnicodeDecodeError:
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
request, cookie)
decoded[key] = cookie[key].decode("latin1", errors="replace")
cookie_str = "{}={}".format(decoded.pop("name"), decoded.pop("value"))
for key, value in decoded.items(): # path, domain
cookie_str += "; {}={}".format(key.capitalize(), value)
return cookie_str
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
cookie_list = [
{'name': k, 'value': v}
for k, v in request.cookies.items()
]
else:
cookie_list = request.cookies
"""
Extract cookies from a Request. Values from the `Request.cookies` attribute
take precedence over values from the `Cookie` request header.
"""
def get_cookies_from_header(jar, request):
cookie_header = request.headers.get("Cookie")
if not cookie_header:
return []
cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
cookie_list_unicode = []
for cookie_bytes in cookie_gen_bytes:
try:
cookie_unicode = cookie_bytes.decode("utf8")
except UnicodeDecodeError:
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
request, cookie_bytes)
cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
cookie_list_unicode.append(cookie_unicode)
response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
return jar.make_cookies(response, request)
cookies = [self._format_cookie(x) for x in cookie_list]
headers = {'Set-Cookie': cookies}
response = Response(request.url, headers=headers)
def get_cookies_from_attribute(jar, request):
if not request.cookies:
return []
elif isinstance(request.cookies, dict):
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
else:
cookies = request.cookies
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
response = Response(request.url, headers={"Set-Cookie": formatted})
return jar.make_cookies(response, request)
return jar.make_cookies(response, request)
return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)

2
scrapy/downloadermiddlewares/decompression.py
View File

@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes
logger = logging.getLogger(__name__)
class DecompressionMiddleware(object):
class DecompressionMiddleware:
""" This middleware tries to recognise and extract the possibly compressed
responses that may arrive. """

2
scrapy/downloadermiddlewares/defaultheaders.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
from scrapy.utils.python import without_none_values
class DefaultHeadersMiddleware(object):
class DefaultHeadersMiddleware:
def __init__(self, headers):
self._headers = headers

2
scrapy/downloadermiddlewares/downloadtimeout.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst
from scrapy import signals
class DownloadTimeoutMiddleware(object):
class DownloadTimeoutMiddleware:
def __init__(self, timeout=180):
self._timeout = timeout

Some files were not shown because too many files have changed in this diff Show More