RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into extending-s3-files-store

This commit is contained in:
Daniel Graña 2017-12-31 16:44:38 -03:00 committed by GitHub
parent 11cf6ad425 aa83e159c9
commit 2dee191374
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
183 changed files with 4440 additions and 799 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.bumpversion.cfg
View File

@ -1,7 +1,8 @@
[bumpversion]
current_version = 1.3.2
current_version = 1.5.0
commit = True
tag = True
tag_name = {new_version}
[bumpversion:file:scrapy/VERSION]

34
.travis.yml
View File

@ -11,32 +11,32 @@ matrix:
env: TOXENV=py27
- python: 2.7
env: TOXENV=jessie
- python: 3.3
env: TOXENV=py33
- python: 2.7
env: TOXENV=pypy
- python: 2.7
env: TOXENV=pypy3
- python: 3.4
env: TOXENV=py34
- python: 3.5
env: TOXENV=py35
- python: 3.6
env: TOXENV=py36
- python: 2.7
env: TOXENV=pypy
- python: 3.6
env: TOXENV=docs
allow_failures:
- python: 2.7
env: TOXENV=pypy
install:
- |
if [ "$TOXENV" = "pypy" ]; then
export PYENV_ROOT="$HOME/.pyenv"
if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
pushd "$PYENV_ROOT" && git pull && popd
else
rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
fi
# get latest PyPy from pyenv directly (thanks to natural version sort option -V)
export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1`
"$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION"
virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION"
export PYPY_VERSION="pypy-5.9-linux_x86_64-portable"
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
if [ "$TOXENV" = "pypy3" ]; then
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
tar -jxf ${PYPY_VERSION}.tar.bz2
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
fi
- pip install -U tox twine wheel codecov

4
CONTRIBUTING.md
View File

@ -1,6 +1,6 @@
The guidelines for contributing are available here:
http://doc.scrapy.org/en/master/contributing.html
https://doc.scrapy.org/en/master/contributing.html
Please do not abuse the issue tracker for support questions.
If your issue topic can be rephrased to "How to ...?", please use the
support channels to get it answered: http://scrapy.org/community/
support channels to get it answered: https://scrapy.org/community/

2
INSTALL
View File

@ -1,4 +1,4 @@
For information about installing Scrapy see:
* docs/intro/install.rst (local file)
* http://doc.scrapy.org/en/latest/intro/install.html (online version)
* https://doc.scrapy.org/en/latest/intro/install.html (online version)

6
LICENSE
View File

@ -4,10 +4,10 @@ All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

35
README.rst
View File

@ -6,8 +6,12 @@ Scrapy
:target: https://pypi.python.org/pypi/Scrapy
:alt: PyPI Version
.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg
:target: https://pypi.python.org/pypi/Scrapy
:alt: Supported Python Versions
.. image:: https://img.shields.io/travis/scrapy/scrapy/master.svg
:target: http://travis-ci.org/scrapy/scrapy
:target: https://travis-ci.org/scrapy/scrapy
:alt: Build Status
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
@ -15,7 +19,7 @@ Scrapy
:alt: Wheel Status
.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
:target: http://codecov.io/github/scrapy/scrapy?branch=master
:target: https://codecov.io/github/scrapy/scrapy?branch=master
:alt: Coverage report
.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg
@ -31,12 +35,12 @@ crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
For more information including a list of features check the Scrapy homepage at:
http://scrapy.org
https://scrapy.org
Requirements
============
* Python 2.7 or Python 3.3+
* Python 2.7 or Python 3.4+
* Works on Linux, Windows, Mac OSX, BSD
Install
@ -47,29 +51,28 @@ The quick way::
pip install scrapy
For more details see the install section in the documentation:
http://doc.scrapy.org/en/latest/intro/install.html
Releases
========
You can download the latest stable and development releases from:
http://scrapy.org/download/
https://doc.scrapy.org/en/latest/intro/install.html
Documentation
=============
Documentation is available online at http://doc.scrapy.org/ and in the ``docs``
Documentation is available online at https://doc.scrapy.org/ and in the ``docs``
directory.
Releases
========
You can find release notes at https://doc.scrapy.org/en/latest/news.html
Community (blog, twitter, mail list, IRC)
=========================================
See http://scrapy.org/community/
See https://scrapy.org/community/
Contributing
============
See http://doc.scrapy.org/en/master/contributing.html
See https://doc.scrapy.org/en/master/contributing.html
Code of Conduct
---------------
@ -83,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
Companies using Scrapy
======================
See http://scrapy.org/companies/
See https://scrapy.org/companies/
Commercial Support
==================
See http://scrapy.org/support/
See https://scrapy.org/support/

4
artwork/README.rst
View File

@ -10,10 +10,10 @@ scrapy-logo.jpg
Main Scrapy logo, in JPEG format.
qlassik.zip
qlassik.zip
-----------
Font used for Scrapy logo. Homepage: http://www.dafont.com/qlassik.font
Font used for Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
scrapy-blog.logo.xcf
--------------------

6
codecov.yml Normal file
View File

@ -0,0 +1,6 @@
comment:
layout: "header, diff, tree"
coverage:
status:
project: false

6
debian/control vendored
View File

@ -4,7 +4,7 @@ Priority: optional
Maintainer: Scrapinghub Team <info@scrapinghub.com>
Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
Standards-Version: 3.8.4
Homepage: http://scrapy.org/
Homepage: https://scrapy.org/
Package: scrapy
Architecture: all
@ -15,6 +15,6 @@ Conflicts: python-scrapy, scrapy-0.25
Provides: python-scrapy, scrapy-0.25
Description: Python web crawling and web scraping framework
Scrapy is a fast high-level web crawling and web scraping framework,
used to crawl websites and extract structured data from their pages.
It can be used for a wide range of purposes, from data mining to
used to crawl websites and extract structured data from their pages.
It can be used for a wide range of purposes, from data mining to
monitoring and automated testing.

8
debian/copyright vendored
View File

@ -1,6 +1,6 @@
This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
It was downloaded from http://scrapy.org
It was downloaded from https://scrapy.org
Upstream Author: Scrapy Developers
@ -14,10 +14,10 @@ All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

6
docs/Makefile
View File

@ -10,7 +10,8 @@ PAPER =
SOURCES =
SHELL = /bin/bash
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees -D latex_paper_size=$(PAPER) \
ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \
-D latex_elements.papersize=$(PAPER) \
$(SPHINXOPTS) . build/$(BUILDER) $(SOURCES)
.PHONY: help update build html htmlhelp clean
@ -82,7 +83,8 @@ pydoc-topics: build
"into the Lib/ directory"
htmlview: html
$(PYTHON) -c "import webbrowser; webbrowser.open('build/html/index.html')"
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
os.path.realpath('build/html/index.html'))"
clean:
-rm -rf build/*

4
docs/README.rst
View File

@ -11,11 +11,11 @@ Setup the environment
---------------------
To compile the documentation you need Sphinx Python library. To install it
and all its dependencies run
and all its dependencies run the following command from this dir
::
pip install 'Sphinx >= 1.3'
pip install -r requirements.txt
Compile the documentation

4
docs/conf.py
View File

@ -191,8 +191,8 @@ htmlhelp_basename = 'Scrapydoc'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
latex_documents = [
('index', 'Scrapy.tex', ur'Scrapy Documentation',
ur'Scrapy developers', 'manual'),
('index', 'Scrapy.tex', u'Scrapy Documentation',
u'Scrapy developers', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of

91
docs/contributing.rst
View File

@ -7,7 +7,7 @@ Contributing to Scrapy
.. important::
Double check you are reading the most recent version of this document at
http://doc.scrapy.org/en/master/contributing.html
https://doc.scrapy.org/en/master/contributing.html
There are many ways to contribute to Scrapy. Here are some of them:
@ -19,12 +19,16 @@ There are many ways to contribute to Scrapy. Here are some of them:
the guidelines detailed in `Reporting bugs`_ below.
* Submit patches for new functionality and/or bug fixes. Please read
`Writing patches`_ and `Submitting patches`_ below for details on how to
:ref:`writing-patches` and `Submitting patches`_ below for details on how to
write and submit a patch.
* Join the `scrapy-users`_ mailing list and share your ideas on how to
* Join the `Scrapy subreddit`_ and share your ideas on how to
improve Scrapy. We're always open to suggestions.
* Answer Scrapy questions at
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__.
Reporting bugs
==============
@ -40,13 +44,18 @@ guidelines when reporting a new bug.
* check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
well-known question
* check the `open issues`_ to see if it has already been reported. If it has,
don't dismiss the report but check the ticket history and comments, you may
find additional useful information to contribute.
* if you have a general question about scrapy usage, please ask it at
`Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
(use "scrapy" tag).
* search the `scrapy-users`_ list to see if it has been discussed there, or
if you're not sure if what you're seeing is a bug. You can also ask in the
`#scrapy` IRC channel.
* check the `open issues`_ to see if it has already been reported. If it has,
don't dismiss the report, but check the ticket history and comments. If you
have additional useful information, please leave a comment, or consider
:ref:`sending a pull request <writing-patches>` with a fix.
* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
been discussed there, or if you're not sure if what you're seeing is a bug.
You can also ask in the `#scrapy` IRC channel.
* write **complete, reproducible, specific bug reports**. The smaller the test
case, the better. Remember that other developers won't have your project to
@ -54,12 +63,20 @@ guidelines when reporting a new bug.
it. See for example StackOverflow's guide on creating a
`Minimal, Complete, and Verifiable example`_ exhibiting the issue.
* the most awesome way to provide a complete reproducible example is to
send a pull request which adds a failing test case to the
Scrapy testing suite (see :ref:`submitting-patches`).
This is helpful even if you don't have an intention to
fix the issue yourselves.
* include the output of ``scrapy version -v`` so developers working on your bug
know exactly which version and platform it occurred on, which is often very
helpful for reproducing it, or knowing if it was already fixed.
.. _Minimal, Complete, and Verifiable example: https://stackoverflow.com/help/mcve
.. _writing-patches:
Writing patches
===============
@ -83,6 +100,8 @@ Well-written patches should:
the documentation changes in the same patch. See `Documentation policies`_
below.
.. _submitting-patches:
Submitting patches
==================
@ -98,13 +117,31 @@ patch, but it's always good to have a patch ready to illustrate your arguments
and show that you have put some additional thought into the subject. A good
starting point is to send a pull request on GitHub. It can be simple enough to
illustrate your idea, and leave documentation/tests for later, after the idea
has been validated and proven useful. Alternatively, you can send an email to
`scrapy-users`_ to discuss your idea first.
has been validated and proven useful. Alternatively, you can start a
conversation in the `Scrapy subreddit`_ to discuss your idea first.
Sometimes there is an existing pull request for the problem you'd like to
solve, which is stalled for some reason. Often the pull request is in a
right direction, but changes are requested by Scrapy maintainers, and the
original pull request author haven't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
not considered rude as soon as the original author is acknowledged by keeping
his/her commits.
You can pull an existing pull request to a local branch
by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE``
(replace 'upstream' with a remote name for scrapy repository,
``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE``
with a name of the branch you want to create locally).
See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally.
When writing GitHub pull requests, try to keep titles short but descriptive.
E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
prefer "Fix hanging when exception occurs in start_requests (#411)"
instead of "Fix for #411".
Complete titles make it easy to skim through the issue tracker.
instead of "Fix for #411". Complete titles make it easy to skim through
the issue tracker.
Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
removal, etc) in separate commits than functional changes. This will make pull
@ -121,30 +158,29 @@ Scrapy:
* It's OK to use lines longer than 80 chars if it improves the code
readability.
* Don't put your name in the code you contribute. Our policy is to keep
the contributor's name in the `AUTHORS`_ file distributed with Scrapy.
Scrapy Contrib
==============
Scrapy contrib shares a similar rationale as Django contrib, which is explained
in `this post <https://jacobian.org/writing/what-is-django-contrib/>`_. If you
are working on a new functionality, please follow that rationale to decide
whether it should be a Scrapy contrib. If unsure, you can ask in
`scrapy-users`_.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
See https://help.github.com/articles/setting-your-username-in-git/ for
setup instructions.
Documentation policies
======================
* **Don't** use docstrings for documenting classes, or methods which are
already documented in the official (sphinx) documentation. For example, the
:meth:`ItemLoader.add_value` method should be documented in the sphinx
documentation, not its docstring.
already documented in the official (sphinx) documentation. Alternatively,
**do** provide a docstring, but make sure sphinx documentation uses
autodoc_ extension to pull the docstring. For example, the
:meth:`ItemLoader.add_value` method should be either
documented only in the sphinx documentation (not it a docstring), or
it should have a docstring which is pulled to sphinx documentation using
autodoc_ extension.
* **Do** use docstrings for documenting functions not present in the official
(sphinx) documentation, such as functions from ``scrapy.utils`` package and
its sub-modules.
.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
Tests
=====
@ -197,6 +233,7 @@ And their unit-tests are in::
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://reddit.com/r/scrapy
.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests

9
docs/faq.rst
View File

@ -21,7 +21,7 @@ Python code.
In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like
comparing `jinja2`_ to `Django`_.
.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: http://lxml.de/
.. _jinja2: http://jinja.pocoo.org/
.. _Django: https://www.djangoproject.com/
@ -69,12 +69,15 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
What Python versions does Scrapy support?
-----------------------------------------
Scrapy is supported under Python 2.7 and Python 3.3+.
Scrapy is supported under Python 2.7 and Python 3.4+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Python 2.6 support was dropped starting at Scrapy 0.20.
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
.. note::
Python 3 is not yet supported on Windows.
For Python 3 support on Windows, it is recommended to use
Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
Did Scrapy "steal" X from Django?
---------------------------------

10
docs/index.rst
View File

@ -13,14 +13,14 @@ Having trouble? We'd like to help!
* Try the :doc:`FAQ <faq>` -- it's got answers to some common questions.
* Looking for specific information? Try the :ref:`genindex` or :ref:`modindex`.
* Ask or search questions in `StackOverflow using the scrapy tag`_,
* Search for information in the `archives of the scrapy-users mailing list`_, or
`post a question`_.
* Ask or search questions in `StackOverflow using the scrapy tag`_.
* Ask or search questions in the `Scrapy subreddit`_.
* Search for questions on the archives of the `scrapy-users mailing list`_.
* Ask a question in the `#scrapy IRC channel`_,
* Report bugs with Scrapy in our `issue tracker`_.
.. _archives of the scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
.. _post a question: https://groups.google.com/forum/#!forum/scrapy-users
.. _scrapy-users mailing list: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
.. _StackOverflow using the scrapy tag: https://stackoverflow.com/tags/scrapy
.. _#scrapy IRC channel: irc://irc.freenode.net/scrapy
.. _issue tracker: https://github.com/scrapy/scrapy/issues

116
docs/intro/install.rst
View File

@ -7,14 +7,26 @@ Installation guide
Installing Scrapy
=================
Scrapy runs on Python 2.7 and Python 3.3 or above
(except on Windows where Python 3 is not supported yet).
Scrapy runs on Python 2.7 and Python 3.4 or above
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
If you’re already familiar with installation of Python packages,
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
and OS X.
To install Scrapy using ``conda``, run::
conda install -c conda-forge scrapy
Alternatively, if you’re already familiar with installation of Python packages,
you can install Scrapy and its dependencies from PyPI with::
pip install Scrapy
Note that sometimes this may require solving compilation issues for some Scrapy
dependencies depending on your operating system, so be sure to check the
:ref:`intro-install-platform-notes`.
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
to avoid conflicting with your system packages.
@ -96,7 +108,7 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def
.. _virtualenv: https://virtualenv.pypa.io
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.io/en/latest/install.html
.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html
.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
@ -105,52 +117,28 @@ Python virtualenvs can be created to use Python 2 by default, or Python 3 by def
Platform specific installation notes
====================================
.. _intro-install-windows:
Windows
-------
* Install Python 2.7 from https://www.python.org/downloads/
Though it's possible to install Scrapy on Windows using pip, we recommend you
to install `Anaconda`_ or `Miniconda`_ and use the package from the
`conda-forge`_ channel, which will avoid most installation issues.
You need to adjust ``PATH`` environment variable to include paths to
the Python executable and additional scripts. The following paths need to be
added to ``PATH``::
Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
C:\Python27\;C:\Python27\Scripts\;
conda install -c conda-forge scrapy
To update the ``PATH`` open a Command prompt and run::
c:\python27\python.exe c:\python27\tools\scripts\win_add2path.py
.. _intro-install-ubuntu:
Close the command prompt window and reopen it so changes take effect, run the
following command and check it shows the expected Python version::
python --version
* Install `pywin32` from http://sourceforge.net/projects/pywin32/
Be sure you download the architecture (win32 or amd64) that matches your system
* *(Only required for Python<2.7.9)* Install `pip`_ from
https://pip.pypa.io/en/latest/installing/
Now open a Command prompt to check ``pip`` is installed correctly::
pip --version
* At this point Python 2.7 and ``pip`` package manager must be working, let's
install Scrapy::
pip install Scrapy
.. note::
Python 3 is not supported on Windows. This is because Scrapy core requirement Twisted does not support
Python 3 on Windows.
Ubuntu 12.04 or above
Ubuntu 14.04 or above
---------------------
Scrapy is currently tested with recent-enough versions of lxml,
twisted and pyOpenSSL, and is compatible with recent Ubuntu distributions.
But it should support older versions of Ubuntu too, like Ubuntu 12.04,
But it should support older versions of Ubuntu too, like Ubuntu 14.04,
albeit with potential issues with TLS connections.
**Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
@ -176,10 +164,12 @@ you can install Scrapy with ``pip`` after that::
pip install scrapy
.. note::
The same non-python dependencies can be used to install Scrapy in Debian
Wheezy (7.0) and above.
The same non-Python dependencies can be used to install Scrapy in Debian
Jessie (8.0) and above.
.. _intro-install-macos:
Mac OS X
--------
@ -199,7 +189,7 @@ solutions:
that doesn't conflict with the rest of your system. Here's how to do it using
the `homebrew`_ package manager:
* Install `homebrew`_ following the instructions in http://brew.sh/
* Install `homebrew`_ following the instructions in https://brew.sh/
* Update your ``PATH`` variable to state that homebrew packages should be
used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
@ -234,27 +224,31 @@ After any of these workarounds you should be able to install Scrapy::
pip install Scrapy
Anaconda
--------
PyPy
----
We recommend using the latest PyPy version. The version tested is 5.9.0.
For PyPy3, only Linux installation was tested.
Using Anaconda is an alternative to using a virtualenv and installing with ``pip``.
Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
On OS X, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
that is to ``brew install openssl`` and then export the flags that this command
recommends (only needed when installing scrapy). Installing on Linux has no special
issues besides installing build dependencies.
Installing scrapy with PyPy on Windows is not tested.
.. note::
You can check that scrapy is installed correctly by running ``scrapy bench``.
If this command gives errors such as
``TypeError: ... got 2 unexpected keyword arguments``, this means
that setuptools was unable to pick up one PyPy-specific dependency.
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
For Windows users, or if you have issues installing through ``pip``, this is
the recommended way to install Scrapy.
If you already have `Anaconda`_ or `Miniconda`_ installed, the `conda-forge`_
community have up-to-date packages for Linux, Windows and OS X.
To install Scrapy using ``conda``, run::
conda install -c conda-forge scrapy
.. _Python: https://www.python.org/
.. _pip: https://pip.pypa.io/en/latest/installing/
.. _Control Panel: https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx
.. _lxml: http://lxml.de/
.. _parsel: https://pypi.python.org/pypi/parsel
.. _w3lib: https://pypi.python.org/pypi/w3lib
@ -263,9 +257,9 @@ To install Scrapy using ``conda``, run::
.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
.. _setuptools: https://pypi.python.org/pypi/setuptools
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
.. _homebrew: http://brew.sh/
.. _zsh: http://www.zsh.org/
.. _Scrapinghub: http://scrapinghub.com
.. _Anaconda: http://docs.continuum.io/anaconda/index
.. _Miniconda: http://conda.pydata.org/docs/install/quick.html
.. _conda-forge: https://conda-forge.github.io/
.. _homebrew: https://brew.sh/
.. _zsh: https://www.zsh.org/
.. _Scrapinghub: https://scrapinghub.com
.. _Anaconda: https://docs.anaconda.com/anaconda/
.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
.. _conda-forge: https://conda-forge.org/

4
docs/intro/overview.rst
View File

@ -160,8 +160,8 @@ The next steps for you are to :ref:`install Scrapy <intro-install>`,
a full-blown Scrapy project and `join the community`_. Thanks for your
interest!
.. _join the community: http://scrapy.org/community/
.. _join the community: https://scrapy.org/community/
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/gp/advertising/api/detail/main.html
.. _Amazon S3: https://aws.amazon.com/s3/
.. _Sitemaps: http://www.sitemaps.org
.. _Sitemaps: https://www.sitemaps.org/index.html

6
docs/intro/tutorial.rst
View File

@ -34,7 +34,7 @@ list of Python resources for non-programmers`_.
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
.. _Dive Into Python 3: http://www.diveintopython3.net
.. _Python Tutorial: https://docs.python.org/3/tutorial
.. _Learn Python The Hard Way: http://learnpythonthehardway.org/book/
.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/
Creating a project
@ -54,6 +54,8 @@ This will create a ``tutorial`` directory with the following contents::
__init__.py
items.py # project items definition file
middlewares.py # project middlewares file
pipelines.py # project pipelines file
@ -452,7 +454,7 @@ For historic reasons, Scrapy appends to a given file instead of overwriting
its contents. If you run this command twice without removing the file
before the second time, you'll end up with a broken JSON file.
You can also used other formats, like `JSON Lines`_::
You can also use other formats, like `JSON Lines`_::
scrapy crawl quotes -o quotes.jl

332
docs/news.rst
View File

@ -3,13 +3,323 @@
Release notes
=============
Scrapy 1.5.0 (2017-12-29)
-------------------------
This release brings small new features and improvements across the codebase.
Some highlights:
* Google Cloud Storage is supported in FilesPipeline and ImagesPipeline.
* Crawling with proxy servers becomes more efficient, as connections
to proxies can be reused now.
* Warnings, exception and logging messages are improved to make debugging
easier.
* ``scrapy parse`` command now allows to set custom request meta via
``--meta`` argument.
* Compatibility with Python 3.6, PyPy and PyPy3 is improved;
PyPy and PyPy3 are now supported officially, by running tests on CI.
* Better default handling of HTTP 308, 522 and 524 status codes.
* Documentation is improved, as usual.
Backwards Incompatible Changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Scrapy 1.5 drops support for Python 3.3.
* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
**This is technically backwards-incompatible**; override
:setting:`USER_AGENT` if you relied on old value.
* Logging of settings overridden by ``custom_settings`` is fixed;
**this is technically backwards-incompatible** because the logger
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're
parsing Scrapy logs, please update your log parsers (:issue:`1343`).
* LinkExtractor now ignores ``m4v`` extension by default, this is change
in behavior.
* 522 and 524 status codes are added to ``RETRY_HTTP_CODES`` (:issue:`2851`)
New features
~~~~~~~~~~~~
- Support ``<link>`` tags in ``Response.follow`` (:issue:`2785`)
- Support for ``ptpython`` REPL (:issue:`2654`)
- Google Cloud Storage support for FilesPipeline and ImagesPipeline
(:issue:`2923`).
- New ``--meta`` option of the "scrapy parse" command allows to pass additional
request.meta (:issue:`2883`)
- Populate spider variable when using ``shell.inspect_response`` (:issue:`2812`)
- Handle HTTP 308 Permanent Redirect (:issue:`2844`)
- Add 522 and 524 to ``RETRY_HTTP_CODES`` (:issue:`2851`)
- Log versions information at startup (:issue:`2857`)
- ``scrapy.mail.MailSender`` now works in Python 3 (it requires Twisted 17.9.0)
- Connections to proxy servers are reused (:issue:`2743`)
- Add template for a downloader middleware (:issue:`2755`)
- Explicit message for NotImplementedError when parse callback not defined
(:issue:`2831`)
- CrawlerProcess got an option to disable installation of root log handler
(:issue:`2921`)
- LinkExtractor now ignores ``m4v`` extension by default
- Better log messages for responses over :setting:`DOWNLOAD_WARNSIZE` and
:setting:`DOWNLOAD_MAXSIZE` limits (:issue:`2927`)
- Show warning when a URL is put to ``Spider.allowed_domains`` instead of
a domain (:issue:`2250`).
Bug fixes
~~~~~~~~~
- Fix logging of settings overridden by ``custom_settings``;
**this is technically backwards-incompatible** because the logger
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please
update your log parsers if needed (:issue:`1343`)
- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
**This is technically backwards-incompatible**; override
:setting:`USER_AGENT` if you relied on old value.
- Fix PyPy and PyPy3 test failures, support them officially
(:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`,
:issue:`3048`)
- Fix DNS resolver when ``DNSCACHE_ENABLED=False`` (:issue:`2811`)
- Add ``cryptography`` for Debian Jessie tox test env (:issue:`2848`)
- Add verification to check if Request callback is callable (:issue:`2766`)
- Port ``extras/qpsclient.py`` to Python 3 (:issue:`2849`)
- Use getfullargspec under the scenes for Python 3 to stop DeprecationWarning
(:issue:`2862`)
- Update deprecated test aliases (:issue:`2876`)
- Fix ``SitemapSpider`` support for alternate links (:issue:`2853`)
Docs
~~~~
- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY``
setting. (:issue:`2756`)
- Update Contributing docs, document new support channels
(:issue:`2762`, issue:`3038`)
- Include references to Scrapy subreddit in the docs
- Fix broken links; use https:// for external links
(:issue:`2978`, :issue:`2982`, :issue:`2958`)
- Document CloseSpider extension better (:issue:`2759`)
- Use ``pymongo.collection.Collection.insert_one()`` in MongoDB example
(:issue:`2781`)
- Spelling mistake and typos
(:issue:`2828`, :issue:`2837`, :issue:`#2884`, :issue:`2924`)
- Clarify ``CSVFeedSpider.headers`` documentation (:issue:`2826`)
- Document ``DontCloseSpider`` exception and clarify ``spider_idle``
(:issue:`2791`)
- Update "Releases" section in README (:issue:`2764`)
- Fix rst syntax in ``DOWNLOAD_FAIL_ON_DATALOSS`` docs (:issue:`2763`)
- Small fix in description of startproject arguments (:issue:`2866`)
- Clarify data types in Response.body docs (:issue:`2922`)
- Add a note about ``request.meta['depth']`` to DepthMiddleware docs (:issue:`2374`)
- Add a note about ``request.meta['dont_merge_cookies']`` to CookiesMiddleware
docs (:issue:`2999`)
- Up-to-date example of project structure (:issue:`2964`, :issue:`2976`)
- A better example of ItemExporters usage (:issue:`2989`)
- Document ``from_crawler`` methods for spider and downloader middlewares
(:issue:`3019`)
Scrapy 1.4.0 (2017-05-18)
-------------------------
Scrapy 1.4 does not bring that many breathtaking new features
but quite a few handy improvements nonetheless.
Scrapy now supports anonymous FTP sessions with customizable user and
password via the new :setting:`FTP_USER` and :setting:`FTP_PASSWORD` settings.
And if you're using Twisted version 17.1.0 or above, FTP is now available
with Python 3.
There's a new :meth:`response.follow <scrapy.http.TextResponse.follow>` method
for creating requests; **it is now a recommended way to create Requests
in Scrapy spiders**. This method makes it easier to write correct
spiders; ``response.follow`` has several advantages over creating
``scrapy.Request`` objects directly:
* it handles relative URLs;
* it works properly with non-ascii URLs on non-UTF8 pages;
* in addition to absolute and relative URLs it supports Selectors;
for ``<a>`` elements it can also extract their href values.
For example, instead of this::
for href in response.css('li.page a::attr(href)').extract():
url = response.urljoin(href)
yield scrapy.Request(url, self.parse, encoding=response.encoding)
One can now write this::
for a in response.css('li.page a'):
yield response.follow(a, self.parse)
Link extractors are also improved. They work similarly to what a regular
modern browser would do: leading and trailing whitespace are removed
from attributes (think ``href=" http://example.com"``) when building
``Link`` objects. This whitespace-stripping also happens for ``action``
attributes with ``FormRequest``.
**Please also note that link extractors do not canonicalize URLs by default
anymore.** This was puzzling users every now and then, and it's not what
browsers do in fact, so we removed that extra transformation on extractred
links.
For those of you wanting more control on the ``Referer:`` header that Scrapy
sends when following links, you can set your own ``Referrer Policy``.
Prior to Scrapy 1.4, the default ``RefererMiddleware`` would simply and
blindly set it to the URL of the response that generated the HTTP request
(which could leak information on your URL seeds).
By default, Scrapy now behaves much like your regular browser does.
And this policy is fully customizable with W3C standard values
(or with something really custom of your own if you wish).
See :setting:`REFERRER_POLICY` for details.
To make Scrapy spiders easier to debug, Scrapy logs more stats by default
in 1.4: memory usage stats, detailed retry stats, detailed HTTP error code
stats. A similar change is that HTTP cache path is also visible in logs now.
Last but not least, Scrapy now has the option to make JSON and XML items
more human-readable, with newlines between items and even custom indenting
offset, using the new :setting:`FEED_EXPORT_INDENT` setting.
Enjoy! (Or read on for the rest of changes in this release.)
Deprecations and Backwards Incompatible Changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
(:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
**warning, this is technically backwards-incompatible**
- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
**this is technically backwards-incompatible** so please check if you have
any non-default ``MEMUSAGE_***`` options set.
- ``EDITOR`` environment variable now takes precedence over ``EDITOR``
option defined in settings.py (:issue:`1829`); Scrapy default settings
no longer depend on environment variables. **This is technically a backwards
incompatible change**.
- ``Spider.make_requests_from_url`` is deprecated
(:issue:`1728`, fixes :issue:`1495`).
New Features
~~~~~~~~~~~~
- Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
- Support `brotli`_-compressed content; requires optional `brotlipy`_
(:issue:`2535`)
- New :ref:`response.follow <response-follow-example>` shortcut
for creating requests (:issue:`1940`)
- Added ``flags`` argument and attribute to :class:`Request <scrapy.http.Request>`
objects (:issue:`2047`)
- Support Anonymous FTP (:issue:`2342`)
- Added ``retry/count``, ``retry/max_reached`` and ``retry/reason_count/<reason>``
stats to :class:`RetryMiddleware <scrapy.downloadermiddlewares.retry.RetryMiddleware>`
(:issue:`2543`)
- Added ``httperror/response_ignored_count`` and ``httperror/response_ignored_status_count/<status>``
stats to :class:`HttpErrorMiddleware <scrapy.spidermiddlewares.httperror.HttpErrorMiddleware>`
(:issue:`2566`)
- Customizable :setting:`Referrer policy <REFERRER_POLICY>` in
:class:`RefererMiddleware <scrapy.spidermiddlewares.referer.RefererMiddleware>`
(:issue:`2306`)
- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`)
- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`)
- Warn users when project contains duplicate spider names (fixes :issue:`2181`)
- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
- :ref:`Media downloads <topics-media-pipeline>`, with :class:`FilesPipelines`
or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
- Accept non-complete responses from websites using a new
:setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
- Optional pretty-printing of JSON and XML items via
:setting:`FEED_EXPORT_INDENT` setting (:issue:`2456`, fixes :issue:`1327`)
- Allow dropping fields in ``FormRequest.from_response`` formdata when
``None`` value is passed (:issue:`667`)
- Per-request retry times with the new :reqmeta:`max_retry_times` meta key
(:issue:`2642`)
- ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
(:issue:`2740`)
.. _brotli: https://github.com/google/brotli
.. _brotlipy: https://github.com/python-hyper/brotlipy/
Bug fixes
~~~~~~~~~
- LinkExtractor now strips leading and trailing whitespaces from attributes
(:issue:`2547`, fixes :issue:`1614`)
- Properly handle whitespaces in action attribute in :class:`FormRequest`
(:issue:`2548`)
- Buffer CONNECT response bytes from proxy until all HTTP headers are received
(:issue:`2495`, fixes :issue:`2491`)
- FTP downloader now works on Python 3, provided you use Twisted>=17.1
(:issue:`2599`)
- Use body to choose response type after decompressing content (:issue:`2393`,
fixes :issue:`2145`)
- Always decompress ``Content-Encoding: gzip`` at :class:`HttpCompressionMiddleware
<scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware>` stage (:issue:`2391`)
- Respect custom log level in ``Spider.custom_settings`` (:issue:`2581`,
fixes :issue:`1612`)
- 'make htmlview' fix for macOS (:issue:`2661`)
- Remove "commands" from the command list (:issue:`2695`)
- Fix duplicate Content-Length header for POST requests with empty body (:issue:`2677`)
- Properly cancel large downloads, i.e. above :setting:`DOWNLOAD_MAXSIZE` (:issue:`1616`)
- ImagesPipeline: fixed processing of transparent PNG images with palette
(:issue:`2675`)
Cleanups & Refactoring
~~~~~~~~~~~~~~~~~~~~~~
- Tests: remove temp files and folders (:issue:`2570`),
fixed ProjectUtilsTest on OS X (:issue:`2569`),
use portable pypy for Linux on Travis CI (:issue:`2710`)
- Separate building request from ``_requests_to_follow`` in CrawlSpider (:issue:`2562`)
- Remove “Python 3 progress” badge (:issue:`2567`)
- Add a couple more lines to ``.gitignore`` (:issue:`2557`)
- Remove bumpversion prerelease configuration (:issue:`2159`)
- Add codecov.yml file (:issue:`2750`)
- Set context factory implementation based on Twisted version (:issue:`2577`,
fixes :issue:`2560`)
- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
- Change "localhost" test server certificate (:issue:`2720`)
- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`)
Documentation
~~~~~~~~~~~~~
- Binary mode is required for exporters (:issue:`2564`, fixes :issue:`2553`)
- Mention issue with :meth:`FormRequest.from_response
<scrapy.http.FormRequest.from_response>` due to bug in lxml (:issue:`2572`)
- Use single quotes uniformly in templates (:issue:`2596`)
- Document :reqmeta:`ftp_user` and :reqmeta:`ftp_password` meta keys (:issue:`2587`)
- Removed section on deprecated ``contrib/`` (:issue:`2636`)
- Recommend Anaconda when installing Scrapy on Windows
(:issue:`2477`, fixes :issue:`2475`)
- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
- Rearrange selector sections (:issue:`2705`)
- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
- Mention how to disable request filtering in documentation of
:setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
- Open file in text mode in JSON item writer example (:issue:`2729`)
- Clarify ``allowed_domains`` example (:issue:`2670`)
Scrapy 1.3.3 (2017-03-10)
-------------------------
Bug fixes
~~~~~~~~~
- Make ``SpiderLoader`` raise ``ImportError`` again by default for missing
dependencies and wrong :setting:`SPIDER_MODULES`.
These exceptions were silenced as warnings since 1.3.0.
A new setting is introduced to toggle between warning or exception if needed ;
see :setting:`SPIDER_LOADER_WARN_ONLY` for details.
Scrapy 1.3.2 (2017-02-13)
-------------------------
Bug fixes
~~~~~~~~~
- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`).
- Preserve request class when converting to/from dicts (utils.reqser) (:issue:`2510`).
- Use consistent selectors for author field in tutorial (:issue:`2551`).
- Fix TLS compatibility in Twisted 17+ (:issue:`2558`)
@ -101,6 +411,12 @@ Dependencies & Cleanups
downloader middlewares.
Scrapy 1.2.3 (2017-03-03)
-------------------------
- Packaging fix: disallow unsupported Twisted versions in setup.py
Scrapy 1.2.2 (2016-12-06)
-------------------------
@ -203,7 +519,7 @@ Refactoring
- ``canonicalize_url`` has been moved to `w3lib.url`_ (:issue:`2168`).
.. _w3lib.url: http://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url
.. _w3lib.url: https://w3lib.readthedocs.io/en/latest/w3lib.html#w3lib.url.canonicalize_url
Tests & Requirements
~~~~~~~~~~~~~~~~~~~~
@ -229,6 +545,12 @@ Documentation
- Add StackOverflow as a support channel (:issue:`2257`).
Scrapy 1.1.4 (2017-03-03)
-------------------------
- Packaging fix: disallow unsupported Twisted versions in setup.py
Scrapy 1.1.3 (2016-09-22)
-------------------------
@ -501,6 +823,12 @@ Bugfixes
to same remote host (:issue:`1912`).
Scrapy 1.0.7 (2017-03-03)
-------------------------
- Packaging fix: disallow unsupported Twisted versions in setup.py
Scrapy 1.0.6 (2016-05-04)
-------------------------

2
docs/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
Sphinx>=1.3
sphinx_rtd_theme

1
docs/topics/autothrottle.rst
View File

@ -88,6 +88,7 @@ The settings used to control the AutoThrottle extension are:
* :setting:`AUTOTHROTTLE_ENABLED`
* :setting:`AUTOTHROTTLE_START_DELAY`
* :setting:`AUTOTHROTTLE_MAX_DELAY`
* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`
* :setting:`AUTOTHROTTLE_DEBUG`
* :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
* :setting:`CONCURRENT_REQUESTS_PER_IP`

6
docs/topics/broad-crawls.rst
View File

@ -20,7 +20,7 @@ These are some common properties often found in broad crawls:
* they crawl many domains (often, unbounded) instead of a specific set of sites
* they don't necessarily crawl domains to completion, because it would
* they don't necessarily crawl domains to completion, because it would be
impractical (or impossible) to do so, and instead limit the crawl by time or
number of pages crawled
@ -85,8 +85,8 @@ When doing broad crawls you are often only interested in the crawl rates you
get and any errors found. These stats are reported by Scrapy when using the
``INFO`` log level. In order to save CPU (and log storage requirements) you
should not use ``DEBUG`` log level when preforming large broad crawls in
production. Using ``DEBUG`` level when developing your (broad) crawler may fine
though.
production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.
To set the log level use::

14
docs/topics/commands.rst
View File

@ -55,6 +55,7 @@ structure by default, similar to this::
myproject/
__init__.py
items.py
middlewares.py
pipelines.py
settings.py
spiders/
@ -187,7 +188,7 @@ startproject
Creates a new Scrapy project named ``project_name``, under the ``project_dir``
directory.
If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``myproject``.
If ``project_dir`` wasn't specified, ``project_dir`` will be the same as ``project_name``.
Usage example::
@ -291,12 +292,12 @@ edit
* Syntax: ``scrapy edit <spider>``
* Requires project: *yes*
Edit the given spider using the editor defined in the :setting:`EDITOR`
setting.
Edit the given spider using the editor defined in the ``EDITOR`` environment
variable or (if unset) the :setting:`EDITOR` setting.
This command is provided only as a convenience shortcut for the most common
case, the developer is of course free to choose any tool or IDE to write and
debug his spiders.
debug spiders.
Usage example::
@ -430,6 +431,9 @@ Supported options:
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
response
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
* ``--pipelines``: process items through pipelines
* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
@ -543,7 +547,7 @@ Example::
COMMANDS_MODULE = 'mybot.commands'
.. _Deploying your project: http://scrapyd.readthedocs.org/en/latest/deploy.html
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
Register commands via setup.py entry points
-------------------------------------------

2
docs/topics/debug.rst
View File

@ -142,4 +142,4 @@ available in all future runs should they be necessary again::
For more information, check the :ref:`topics-logging` section.
.. _base tag: http://www.w3schools.com/tags/tag_base.asp
.. _base tag: https://www.w3schools.com/tags/tag_base.asp

12
docs/topics/deploy.rst
View File

@ -50,10 +50,10 @@ them as needed - the configuration is read from the ``scrapy.cfg`` file
just like ``scrapyd-deploy``.
.. _Scrapyd: https://github.com/scrapy/scrapyd
.. _Deploying your project: https://scrapyd.readthedocs.org/en/latest/deploy.html
.. _Scrapy Cloud: http://scrapinghub.com/scrapy-cloud/
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud: https://scrapinghub.com/scrapy-cloud
.. _scrapyd-client: https://github.com/scrapy/scrapyd-client
.. _shub: http://doc.scrapinghub.com/shub.html
.. _scrapyd-deploy documentation: http://scrapyd.readthedocs.org/en/latest/deploy.html
.. _Scrapy Cloud documentation: http://doc.scrapinghub.com/scrapy-cloud.html
.. _Scrapinghub: http://scrapinghub.com/
.. _shub: https://doc.scrapinghub.com/shub.html
.. _scrapyd-deploy documentation: https://scrapyd.readthedocs.io/en/latest/deploy.html
.. _Scrapy Cloud documentation: https://doc.scrapinghub.com/scrapy-cloud.html
.. _Scrapinghub: https://scrapinghub.com/

33
docs/topics/downloader-middleware.rst
View File

@ -157,6 +157,17 @@ more of the following methods:
:param spider: the spider for which this request is intended
:type spider: :class:`~scrapy.spiders.Spider` object
.. method:: from_crawler(cls, crawler)
If present, this classmethod is called to create a middleware instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the middleware. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for middleware to
access them and hook its functionality into Scrapy.
:param crawler: crawler that uses this middleware
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. _topics-downloader-middleware-ref:
Built-in downloader middleware reference
@ -226,6 +237,17 @@ Default: ``True``
Whether to enable the cookies middleware. If disabled, no cookies will be sent
to web servers.
Notice that if the :class:`~scrapy.http.Request`
has ``meta['dont_merge_cookies']`` evaluated to ``True``.
despite the value of :setting:`COOKIES_ENABLED` the cookies will **not** be
sent to web servers and received cookies in
:class:`~scrapy.http.Response` will **not** be merged with the existing
cookies.
For more detailed information see the ``cookies`` parameter in
:class:`~scrapy.http.Request`
.. setting:: COOKIES_DEBUG
COOKIES_DEBUG
@ -645,6 +667,12 @@ HttpCompressionMiddleware
This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites.
This middleware also supports decoding `brotli-compressed`_ responses,
provided `brotlipy`_ is installed.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
.. _brotlipy: https://pypi.python.org/pypi/brotlipy
HttpCompressionMiddleware Settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -846,6 +874,11 @@ Default: ``2``
Maximum number of times to retry, in addition to the first download.
Maximum number of retries can also be specified per-request using
:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.http.Request.meta>`.
When initialized, the :reqmeta:`max_retry_times` meta key takes higher
precedence over the :setting:`RETRY_TIMES` setting.
.. setting:: RETRY_HTTP_CODES
RETRY_HTTP_CODES

4
docs/topics/email.rst
View File

@ -54,10 +54,10 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
:param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER`
setting will be used. If not given, no SMTP authentication will be
performed.
:type smtphost: str
:type smtphost: str or bytes
:param smtppass: the SMTP pass for authentication.
:type smtppass: str
:type smtppass: str or bytes
:param smtpport: the SMTP port to connect to
:type smtpport: int

8
docs/topics/exceptions.rst
View File

@ -39,6 +39,14 @@ For example::
if 'Bandwidth exceeded' in response.body:
raise CloseSpider('bandwidth_exceeded')
DontCloseSpider
---------------
.. exception:: DontCloseSpider
This exception can be raised in a :signal:`spider_idle` signal handler to
prevent the spider from being closed.
IgnoreRequest
-------------

63
docs/topics/exporters.rst
View File

@ -36,38 +36,36 @@ to export
3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal
the end of the exporting process
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses an Item
Exporter to export scraped items to different files, one per spider::
Here you can see an :doc:`Item Pipeline <item-pipeline>` which uses multiple
Item Exporters to group scraped items to different files according to the
value of one of their fields::
from scrapy import signals
from scrapy.exporters import XmlItemExporter
from scrapy.exporters import XmlItemExporter
class XmlExportPipeline(object):
class PerYearXmlExportPipeline(object):
"""Distribute items across multiple XML files according to their 'year' field"""
def __init__(self):
self.files = {}
def open_spider(self, spider):
self.year_to_exporter = {}
@classmethod
def from_crawler(cls, crawler):
pipeline = cls()
crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
return pipeline
def close_spider(self, spider):
for exporter in self.year_to_exporter.values():
exporter.finish_exporting()
exporter.file.close()
def spider_opened(self, spider):
file = open('%s_products.xml' % spider.name, 'w+b')
self.files[spider] = file
self.exporter = XmlItemExporter(file)
self.exporter.start_exporting()
def _exporter_for_item(self, item):
year = item['year']
if year not in self.year_to_exporter:
f = open('{}.xml'.format(year), 'wb')
exporter = XmlItemExporter(f)
exporter.start_exporting()
self.year_to_exporter[year] = exporter
return self.year_to_exporter[year]
def spider_closed(self, spider):
self.exporter.finish_exporting()
file = self.files.pop(spider)
file.close()
def process_item(self, item, spider):
self.exporter.export_item(item)
return item
def process_item(self, item, spider):
exporter = self._exporter_for_item(item)
exporter.export_item(item)
return item
.. _topics-exporters-field-serialization:
@ -140,7 +138,7 @@ output examples, which assume you're exporting these two items::
BaseItemExporter
----------------
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8')
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)
This is the (abstract) base class for all Item Exporters. It provides
support for common features used by all (concrete) Item Exporters, such as
@ -149,7 +147,7 @@ BaseItemExporter
These features can be configured through the constructor arguments which
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`.
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
.. method:: export_item(item)
@ -216,6 +214,15 @@ BaseItemExporter
encoding). Other value types are passed unchanged to the specific
serialization library.
.. attribute:: indent
Amount of spaces used to indent the output on each level. Defaults to ``0``.
* ``indent=None`` selects the most compact representation,
all items in the same line with no indentation
* ``indent<=0`` each item on its own line, no indentation
* ``indent>0`` each item on its own line, indented with the provided numeric value
.. highlight:: none
XmlItemExporter

11
docs/topics/extensions.rst
View File

@ -220,7 +220,6 @@ can be configured with the following settings:
* :setting:`MEMUSAGE_LIMIT_MB`
* :setting:`MEMUSAGE_WARNING_MB`
* :setting:`MEMUSAGE_NOTIFY_MAIL`
* :setting:`MEMUSAGE_REPORT`
* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS`
Memory debugger extension
@ -278,9 +277,11 @@ CLOSESPIDER_ITEMCOUNT
Default: ``0``
An integer which specifies a number of items. If the spider scrapes more than
that amount if items and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``. If zero (or
non set), spiders won't be closed by number of passed items.
that amount and those items are passed by the item pipeline, the
spider will be closed with the reason ``closespider_itemcount``.
Requests which are currently in the downloader queue (up to
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
If zero (or non set), spiders won't be closed by number of passed items.
.. setting:: CLOSESPIDER_PAGECOUNT
@ -372,4 +373,4 @@ For more info see `Debugging in Python`.
This extension only works on POSIX-compliant platforms (ie. not Windows).
.. _Python debugger: https://docs.python.org/2/library/pdb.html
.. _Debugging in Python: http://www.ferg.org/papers/debugging_in_python.html
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/

17
docs/topics/feed-exports.rst
View File

@ -209,6 +209,7 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_ENCODING`
* :setting:`FEED_EXPORT_FIELDS`
* :setting:`FEED_EXPORT_INDENT`
.. currentmodule:: scrapy.extensions.feedexport
@ -266,6 +267,22 @@ If an exporter requires a fixed set of fields (this is the case for
is empty or None, then Scrapy tries to infer field names from the
exported data - currently it uses field names from the first item.
.. setting:: FEED_EXPORT_INDENT
FEED_EXPORT_INDENT
------------------
Default: ``0``
Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``
is a non-negative integer, then array elements and object members will be pretty-printed
with that indent level. An indent level of ``0`` (the default), or negative,
will put each item on a new line. ``None`` selects the most compact representation.
Currently implemented only by :class:`~scrapy.exporters.JsonItemExporter`
and :class:`~scrapy.exporters.XmlItemExporter`, i.e. when you are exporting
to ``.json`` or ``.xml``.
.. setting:: FEED_STORE_EMPTY
FEED_STORE_EMPTY

2
docs/topics/firebug.rst
View File

@ -23,7 +23,7 @@ In this example, we'll show how to use `Firebug`_ to scrape data from the
Project`_ used in the :ref:`tutorial <intro-tutorial>` but with a different
face.
.. _Firebug: http://getfirebug.com
.. _Firebug: https://getfirebug.com/
.. _Google Directory: http://directory.google.com/
.. _Open Directory Project: http://www.dmoz.org

8
docs/topics/firefox.rst
View File

@ -17,7 +17,7 @@ when inspecting the page source is not the original HTML, but a modified one
after applying some browser clean up and executing Javascript code. Firefox,
in particular, is known for adding ``<tbody>`` elements to tables. Scrapy, on
the other hand, does not modify the original page HTML, so you won't be able to
extract any data if you use ``<tbody>`` in your XPath expressions.
extract any data if you use ``<tbody>`` in your XPath expressions.
Therefore, you should keep in mind the following things when working with
Firefox and XPath:
@ -71,11 +71,11 @@ Firecookie
`Firecookie`_ makes it easier to view and manage cookies. You can use this
extension to create a new cookie, delete existing cookies, see a list of cookies
for the current site, manage cookies permissions and a lot more.
for the current site, manage cookies permissions and a lot more.
.. _Firebug: http://getfirebug.com
.. _Firebug: https://getfirebug.com/
.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
.. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/
.. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/
.. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/

9
docs/topics/item-pipeline.rst
View File

@ -107,7 +107,7 @@ format::
class JsonWriterPipeline(object):
def open_spider(self, spider):
self.file = open('items.jl', 'wb')
self.file = open('items.jl', 'w')
def close_spider(self, spider):
self.file.close()
@ -134,7 +134,7 @@ method and how to clean up the resources properly.::
import pymongo
class MongoPipeline(object):
collection_name = 'scrapy_items'
def __init__(self, mongo_uri, mongo_db):
@ -156,7 +156,7 @@ method and how to clean up the resources properly.::
self.client.close()
def process_item(self, item, spider):
self.db[self.collection_name].insert(dict(item))
self.db[self.collection_name].insert_one(dict(item))
return item
.. _MongoDB: https://www.mongodb.org/
@ -208,7 +208,7 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
item["screenshot_filename"] = filename
return item
.. _Splash: http://splash.readthedocs.io/en/stable/
.. _Splash: https://splash.readthedocs.io/en/stable/
.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
Duplicates filter
@ -248,4 +248,3 @@ To activate an Item Pipeline component you must add its class to the
The integer values you assign to classes in this setting determine the
order in which they run: items go through from lower valued to higher
valued classes. It's customary to define these numbers in the 0-1000 range.

2
docs/topics/items.rst
View File

@ -21,7 +21,7 @@ their available fields.
Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
tracks Item instances to help finding memory leaks
tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict

2
docs/topics/jobs.rst
View File

@ -100,4 +100,4 @@ If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
It is ``False`` by default.
.. _pickle: http://docs.python.org/library/pickle.html
.. _pickle: https://docs.python.org/library/pickle.html

11
docs/topics/link-extractors.rst
View File

@ -8,7 +8,7 @@ Link extractors are objects whose only purpose is to extract links from web
pages (:class:`scrapy.http.Response` objects) which will be eventually
followed.
There is ``scrapy.linkextractors import LinkExtractor`` available
There is ``scrapy.linkextractors.LinkExtractor`` available
in Scrapy, but you can create your own custom Link Extractors to suit your
needs by implementing a simple interface.
@ -51,7 +51,7 @@ LxmlLinkExtractor
:synopsis: lxml's HTMLParser-based link extractors
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True)
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True)
LxmlLinkExtractor is the recommended link extractor with handy filtering
options. It is implemented using lxml's robust HTMLParser.
@ -103,7 +103,12 @@ LxmlLinkExtractor
:type attrs: list
:param canonicalize: canonicalize each extracted url (using
w3lib.url.canonicalize_url). Defaults to ``True``.
w3lib.url.canonicalize_url). Defaults to ``False``.
Note that canonicalize_url is meant for duplicate checking;
it can change the URL visible at server side, so the response can be
different for requests with canonicalized and raw URLs. If you're
using LinkExtractor to follow links it is more robust to
keep the default ``canonicalize=False``.
:type canonicalize: boolean
:param unique: whether duplicate filtering should be applied to extracted

4
docs/topics/loaders.rst
View File

@ -518,8 +518,8 @@ a footer of a page that looks something like:
Example::
<footer>
<a class="social" href="http://facebook.com/whatever">Like Us</a>
<a class="social" href="http://twitter.com/whatever">Follow Us</a>
<a class="social" href="https://facebook.com/whatever">Like Us</a>
<a class="social" href="https://twitter.com/whatever">Follow Us</a>
<a class="email" href="mailto:whatever@example.com">Email Us</a>
</footer>

18
docs/topics/logging.rst
View File

@ -27,7 +27,7 @@ Scrapy from scripts as described in :ref:`run-from-script`.
Log levels
==========
Python's builtin logging defines 5 different levels to indicate severity on a
Python's builtin logging defines 5 different levels to indicate the severity of a
given log message. Here are the standard ones, listed in decreasing order:
1. ``logging.CRITICAL`` - for critical errors (highest severity)
@ -47,20 +47,20 @@ level::
There are shortcuts for issuing log messages on any of the standard 5 levels,
and there's also a general ``logging.log`` method which takes a given level as
argument. If you need so, last example could be rewrote as::
argument. If needed, the last example could be rewritten as::
import logging
logging.log(logging.WARNING, "This is a warning")
On top of that, you can create different "loggers" to encapsulate messages (For
example, a common practice it's to create different loggers for every module).
On top of that, you can create different "loggers" to encapsulate messages. (For
example, a common practice is to create different loggers for every module).
These loggers can be configured independently, and they allow hierarchical
constructions.
Last examples use the root logger behind the scenes, which is a top level
The previous examples use the root logger behind the scenes, which is a top level
logger where all messages are propagated to (unless otherwise specified). Using
``logging`` helpers is merely a shortcut for getting the root logger
explicitly, so this is also an equivalent of last snippets::
explicitly, so this is also an equivalent of the last snippets::
import logging
logger = logging.getLogger()
@ -95,14 +95,14 @@ Logging from Spiders
====================
Scrapy provides a :data:`~scrapy.spiders.Spider.logger` within each Spider
instance, that can be accessed and used like this::
instance, which can be accessed and used like this::
import scrapy
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['http://scrapinghub.com']
start_urls = ['https://scrapinghub.com']
def parse(self, response):
self.logger.info('Parse function called on %s', response.url)
@ -118,7 +118,7 @@ Python logger you want. For example::
class MySpider(scrapy.Spider):
name = 'myspider'
start_urls = ['http://scrapinghub.com']
start_urls = ['https://scrapinghub.com']
def parse(self, response):
logger.info('Parse function called on %s', response.url)

41
docs/topics/media-pipeline.rst
View File

@ -15,7 +15,8 @@ typically you'll either use the Files Pipeline or the Images Pipeline.
Both pipelines implement these features:
* Avoid re-downloading media that was downloaded recently
* Specifying where to store the media (filesystem directory, Amazon S3 bucket)
* Specifying where to store the media (filesystem directory, Amazon S3 bucket,
Google Cloud Storage bucket)
The Images Pipeline has a few extra functions for processing images:
@ -116,10 +117,11 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
Supported Storage
=================
File system is currently the only officially supported storage, but there is
also support for storing files in `Amazon S3`_.
File system is currently the only officially supported storage, but there are
also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
.. _Amazon S3: https://aws.amazon.com/s3/
.. _Google Cloud Storage: https://cloud.google.com/storage/
File system storage
-------------------
@ -181,7 +183,26 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
.. _Minio: https://github.com/minio/minio
.. _s3.scality: https://s3.scality.com/
.. _canned ACLs: http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Google Cloud Storage
---------------------
.. setting:: GCS_PROJECT_ID
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings::
IMAGES_STORE = 'gs://bucket/images/'
GCS_PROJECT_ID = 'project_id'
For information about authentication, see this `documentation`_.
.. _documentation: https://cloud.google.com/docs/authentication/production
Usage example
=============
@ -332,6 +353,18 @@ all be dropped because at least one dimension is shorter than the constraint.
By default, there are no size constraints, so all images are processed.
Allowing redirections
---------------------
.. setting:: MEDIA_ALLOW_REDIRECTS
By default media pipelines ignore redirects, i.e. an HTTP redirection
to a media file URL request will mean the media download is considered failed.
To handle media redirections, set this setting to ``True``::
MEDIA_ALLOW_REDIRECTS = True
.. _topics-media-pipeline-override:
Extending the Media Pipelines

10
docs/topics/practices.rst
View File

@ -238,7 +238,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* if possible, use `Google cache`_ to fetch pages, instead of hitting the sites
directly
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alterantive is `scrapoxy`_, a
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
* use a highly distributed downloader that circumvents bans internally, so you
can just focus on parsing clean pages. One example of such downloaders is
@ -248,10 +248,10 @@ If you are still unable to prevent your bot getting banned, consider contacting
`commercial support`_.
.. _Tor project: https://www.torproject.org/
.. _commercial support: http://scrapy.org/support/
.. _ProxyMesh: http://proxymesh.com/
.. _commercial support: https://scrapy.org/support/
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _testspiders: https://github.com/scrapinghub/testspiders
.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
.. _Crawlera: http://scrapinghub.com/crawlera
.. _scrapoxy: http://scrapoxy.io/
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/

29
docs/topics/request-response.rst
View File

@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
Request objects
===============
.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback])
.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
@ -307,6 +307,8 @@ Those are:
* :reqmeta:`proxy`
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* :reqmeta:`referrer_policy`
* :reqmeta:`max_retry_times`
.. reqmeta:: bindaddress
@ -341,6 +343,15 @@ download_fail_on_dataloss
Whether or not to fail on broken responses. See:
:setting:`DOWNLOAD_FAIL_ON_DATALOSS`.
.. reqmeta:: max_retry_times
max_retry_times
---------------
The meta key is used set retry times per request. When initialized, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
.. _topics-request-response-ref-request-subclasses:
Request subclasses
@ -416,7 +427,9 @@ fields with form data from :class:`Response` objects.
:param formdata: fields to override in the form data. If a field was
already present in the response ``<form>`` element, its value is
overridden by the one passed in this parameter.
overridden by the one passed in this parameter. If a value passed in
this parameter is ``None``, the field will not be included in the
request, even if it was present in the response ``<form>`` element.
:type formdata: dict
:param clickdata: attributes to lookup the control clicked. If it's not
@ -512,11 +525,11 @@ Response objects
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
:param body: the response body. It must be str, not unicode, unless you're
using a encoding-aware :ref:`Response subclass
<topics-request-response-ref-response-subclasses>`, such as
:class:`TextResponse`.
:type body: str
:param body: the response body. To access the decoded text as str (unicode
in Python 2) you can use ``response.text`` from an encoding-aware
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
such as :class:`TextResponse`.
:type body: bytes
:param flags: is a list containing the initial values for the
:attr:`Response.flags` attribute. If given, the list will be shallow
@ -721,7 +734,7 @@ HtmlResponse objects
which adds encoding auto-discovering support by looking into the HTML `meta
http-equiv`_ attribute. See :attr:`TextResponse.encoding`.
.. _meta http-equiv: http://www.w3schools.com/TAGS/att_meta_http_equiv.asp
.. _meta http-equiv: https://www.w3schools.com/TAGS/att_meta_http_equiv.asp
XmlResponse objects
-------------------

2
docs/topics/scrapyd.rst
View File

@ -10,4 +10,4 @@ Scrapyd has been moved into a separate project.
Its documentation is now hosted at:
http://scrapyd.readthedocs.org/en/latest/
https://scrapyd.readthedocs.io/en/latest/

19
docs/topics/selectors.rst
View File

@ -36,7 +36,7 @@ documents.
For a complete reference of the selectors API see
:ref:`Selector reference <topics-selectors-ref>`
.. _BeautifulSoup: http://www.crummy.com/software/BeautifulSoup/
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: http://lxml.de/
.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
.. _cssselect: https://pypi.python.org/pypi/cssselect/
@ -86,7 +86,7 @@ To explain how to use the selectors we'll use the `Scrapy shell` (which
provides interactive testing) and an example page located in the Scrapy
documentation server:
http://doc.scrapy.org/en/latest/_static/selectors-sample1.html
https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
.. _topics-selectors-htmlcode:
@ -99,7 +99,7 @@ Here's its HTML code:
First, let's open the shell::
scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html
scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
Then, after the shell loads, you'll have the response available as ``response``
shell variable, and its attached selector in ``response.selector`` attribute.
@ -593,6 +593,9 @@ Built-in Selectors reference
.. module:: scrapy.selector
:synopsis: Selector class
Selector objects
----------------
.. class:: Selector(response=None, text=None, type=None)
An instance of :class:`Selector` is a wrapper over response to select
@ -714,13 +717,9 @@ SelectorList objects
Call the ``.re()`` method for each element in this list and return
their results flattened, as a list of unicode strings.
.. method:: __nonzero__()
returns True if the list is not empty, False otherwise.
Selector examples on HTML response
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
----------------------------------
Here's a couple of :class:`Selector` examples to illustrate several concepts.
In all cases, we assume there is already a :class:`Selector` instantiated with
@ -745,7 +744,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
print node.xpath("@class").extract()
Selector examples on XML response
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
---------------------------------
Here's a couple of examples to illustrate several concepts. In both cases we
assume there is already a :class:`Selector` instantiated with an
@ -767,7 +766,7 @@ assume there is already a :class:`Selector` instantiated with an
.. _removing-namespaces:
Removing namespaces
~~~~~~~~~~~~~~~~~~~
-------------------
When dealing with scraping projects, it is often quite convenient to get rid of
namespaces altogether and just work with element names, to write more

59
docs/topics/settings.rst
View File

@ -655,7 +655,7 @@ Optionally, this can be set per-request basis by using the
circumstances, from server misconfiguration to network errors to data
corruption. It is up to the user to decide if it makes sense to process
broken responses considering they may contain partial or incomplete content.
If setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
If :setting:`RETRY_ENABLED` is ``True`` and this setting is set to ``True``,
the ``ResponseFailed([_DataLoss])`` failure will be retried as usual.
.. setting:: DUPEFILTER_CLASS
@ -674,6 +674,13 @@ override its ``request_fingerprint`` method. This method should accept
scrapy :class:`~scrapy.http.Request` object and return its fingerprint
(a string).
You can disable filtering of duplicate requests by setting
:setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
Be very careful about this however, because you can get into crawling loops.
It's usually a better idea to set the ``dont_filter`` parameter to
``True`` on the specific :class:`~scrapy.http.Request` that should not be
filtered.
.. setting:: DUPEFILTER_DEBUG
DUPEFILTER_DEBUG
@ -689,11 +696,11 @@ Setting :setting:`DUPEFILTER_DEBUG` to ``True`` will make it log all duplicate r
EDITOR
------
Default: `depends on the environment`
Default: ``vi`` (on Unix systems) or the IDLE editor (on Windows)
The editor to use for editing spiders with the :command:`edit` command. It
defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults
to ``vi`` (on Unix systems) or the IDLE editor (on Windows).
The editor to use for editing spiders with the :command:`edit` command.
Additionally, if the ``EDITOR`` environment variable is set, the :command:`edit`
command will prefer it over the default setting.
.. setting:: EXTENSIONS
@ -980,19 +987,6 @@ Example::
See :ref:`topics-extensions-ref-memusage`.
.. setting:: MEMUSAGE_REPORT
MEMUSAGE_REPORT
---------------
Default: ``False``
Scope: ``scrapy.extensions.memusage``
Whether to send a memory usage report after each spider has been closed.
See :ref:`topics-extensions-ref-memusage`.
.. setting:: MEMUSAGE_WARNING_MB
MEMUSAGE_WARNING_MB
@ -1036,7 +1030,7 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option.
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
.. _wget: http://www.gnu.org/software/wget/manual/wget.html
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
.. setting:: REACTOR_THREADPOOL_MAXSIZE
@ -1208,6 +1202,29 @@ Default: ``'scrapy.spiderloader.SpiderLoader'``
The class that will be used for loading spiders, which must implement the
:ref:`topics-api-spiderloader`.
.. setting:: SPIDER_LOADER_WARN_ONLY
SPIDER_LOADER_WARN_ONLY
-----------------------
.. versionadded:: 1.3.3
Default: ``False``
By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
it will fail loudly if there is any ``ImportError`` exception.
But you can choose to silence this exception and turn it into a simple
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
.. note::
Some :ref:`scrapy commands <topics-commands>` run with this setting to ``True``
already (i.e. they will only issue a warning and will not fail)
since they do not actually need to load spider classes to work:
:command:`scrapy runspider <runspider>`,
:command:`scrapy settings <settings>`,
:command:`scrapy startproject <startproject>`,
:command:`scrapy version <version>`.
.. setting:: SPIDER_MIDDLEWARES
SPIDER_MIDDLEWARES
@ -1328,14 +1345,14 @@ Default: ``2083``
Scope: ``spidermiddlewares.urllength``
The maximum URL length to allow for crawled URLs. For more information about
the default value for this setting see: http://www.boutell.com/newfaq/misc/urllength.html
the default value for this setting see: https://boutell.com/newfaq/misc/urllength.html
.. setting:: USER_AGENT
USER_AGENT
----------
Default: ``"Scrapy/VERSION (+http://scrapy.org)"``
Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
The default User-Agent to use when crawling, unless overridden.

14
docs/topics/shell.rst
View File

@ -39,9 +39,9 @@ variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
[settings]
shell = bpython
.. _IPython: http://ipython.org/
.. _IPython installation guide: http://ipython.org/install.html
.. _bpython: http://www.bpython-interpreter.org/
.. _IPython: https://ipython.org/
.. _IPython installation guide: https://ipython.org/install.html
.. _bpython: https://www.bpython-interpreter.org/
Launch the shell
================
@ -142,7 +142,7 @@ Example of shell session
========================
Here's an example of a typical shell session where we start by scraping the
http://scrapy.org page, and then proceed to scrape the https://reddit.com
https://scrapy.org page, and then proceed to scrape the https://reddit.com
page. Finally, we modify the (Reddit) request method to POST and re-fetch it
getting an error. We end the session by typing Ctrl-D (in Unix systems) or
Ctrl-Z in Windows.
@ -154,7 +154,7 @@ shell works.
First, we launch the shell::
scrapy shell 'http://scrapy.org' --nolog
scrapy shell 'https://scrapy.org' --nolog
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
list of available objects and useful shortcuts (you'll notice that these lines
@ -164,7 +164,7 @@ all start with the ``[s]`` prefix)::
[s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc)
[s] crawler <scrapy.crawler.Crawler object at 0x7f07395dd690>
[s] item {}
[s] request <GET http://scrapy.org>
[s] request <GET https://scrapy.org>
[s] response <200 https://scrapy.org/>
[s] settings <scrapy.settings.Settings object at 0x7f07395dd710>
[s] spider <DefaultSpider 'default' at 0x7f0735891690>
@ -182,7 +182,7 @@ After that, we can start playing with the objects::
>>> response.xpath('//title/text()').extract_first()
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
>>> fetch("http://reddit.com")
>>> fetch("https://reddit.com")
>>> response.xpath('//title/text()').extract()
['reddit: the front page of the internet']

10
docs/topics/signals.rst
View File

@ -189,14 +189,20 @@ spider_idle
the engine starts closing the spider. After the spider has finished
closing, the :signal:`spider_closed` signal is sent.
You can, for example, schedule some requests in your :signal:`spider_idle`
handler to prevent the spider from being closed.
You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to
prevent the spider from being closed.
This signal does not support returning deferreds from their handlers.
:param spider: the spider which has gone idle
:type spider: :class:`~scrapy.spiders.Spider` object
.. note:: Scheduling some requests in your :signal:`spider_idle` handler does
**not** guarantee that it can prevent the spider from being closed,
although it sometimes can. That's because the spider may still remain idle
if all the scheduled requests are rejected by the scheduler (e.g. filtered
due to duplication).
spider_error
------------

107
docs/topics/spider-middleware.rst
View File

@ -95,7 +95,7 @@ following methods:
it has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
:class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item`
objects.
:param response: the response which generated this output from the
@ -164,6 +164,17 @@ following methods:
:param spider: the spider to whom the start requests belong
:type spider: :class:`~scrapy.spiders.Spider` object
.. method:: from_crawler(cls, crawler)
If present, this classmethod is called to create a middleware instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the middleware. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for middleware to
access them and hook its functionality into Scrapy.
:param crawler: crawler that uses this middleware
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
@ -188,9 +199,13 @@ DepthMiddleware
.. class:: DepthMiddleware
DepthMiddleware is a scrape middleware used for tracking the depth of each
Request inside the site being scraped. It can be used to limit the maximum
depth to scrape or things like that.
DepthMiddleware is used for tracking the depth of each Request inside the
site being scraped. It works by setting `request.meta['depth'] = 0` whenever
there is no value previously set (usually just the first Request) and
incrementing it by 1 otherwise.
It can be used to limit the maximum depth to scrape, control Request
priority based on their depth, and things like that.
The :class:`DepthMiddleware` can be configured through the following
settings (see the settings documentation for more info):
@ -328,6 +343,90 @@ Default: ``True``
Whether to enable referer middleware.
.. setting:: REFERRER_POLICY
REFERRER_POLICY
^^^^^^^^^^^^^^^
.. versionadded:: 1.4
Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. reqmeta:: referrer_policy
`Referrer Policy`_ to apply when populating Request "Referer" header.
.. note::
You can also set the Referrer Policy per request,
using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
with the same acceptable values as for the ``REFERRER_POLICY`` setting.
Acceptable values for REFERRER_POLICY
*************************************
- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
subclass — a custom policy or one of the built-in ones (see classes below),
- or one of the standard W3C-defined string values,
- or the special ``"scrapy-default"``.
======================================= ========================================================================
String value Class name (as a string)
======================================= ========================================================================
``"scrapy-default"`` (default) :class:`scrapy.spidermiddlewares.referer.DefaultReferrerPolicy`
`"no-referrer"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerPolicy`
`"no-referrer-when-downgrade"`_ :class:`scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy`
`"same-origin"`_ :class:`scrapy.spidermiddlewares.referer.SameOriginPolicy`
`"origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginPolicy`
`"strict-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginPolicy`
`"origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy`
`"strict-origin-when-cross-origin"`_ :class:`scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy`
`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy`
======================================= ========================================================================
.. autoclass:: DefaultReferrerPolicy
.. warning::
Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,
the W3C-recommended value for browsers — will send a non-empty
"Referer" header from any ``http(s)://`` to any ``https://`` URL,
even if the domain is different.
`"same-origin"`_ may be a better choice if you want to remove referrer
information for cross-domain requests.
.. autoclass:: NoReferrerPolicy
.. autoclass:: NoReferrerWhenDowngradePolicy
.. note::
"no-referrer-when-downgrade" policy is the W3C-recommended default,
and is used by major web browsers.
However, it is NOT Scrapy's default referrer policy (see :class:`DefaultReferrerPolicy`).
.. autoclass:: SameOriginPolicy
.. autoclass:: OriginPolicy
.. autoclass:: StrictOriginPolicy
.. autoclass:: OriginWhenCrossOriginPolicy
.. autoclass:: StrictOriginWhenCrossOriginPolicy
.. autoclass:: UnsafeUrlPolicy
.. warning::
"unsafe-url" policy is NOT recommended.
.. _Referrer Policy: https://www.w3.org/TR/referrer-policy
.. _"no-referrer": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
.. _"no-referrer-when-downgrade": https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
.. _"same-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
.. _"origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
.. _"strict-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
.. _"origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
.. _"strict-origin-when-cross-origin": https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
.. _"unsafe-url": https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
UrlLengthMiddleware
-------------------

12
docs/topics/spiders.rst
View File

@ -81,6 +81,9 @@ scrapy.Spider
specified in this list (or their subdomains) won't be followed if
:class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
Let's say your target url is ``https://www.example.com/1.html``,
then add ``'example.com'`` to the list.
.. attribute:: start_urls
A list of URLs where the spider will begin to crawl from, when no
@ -575,8 +578,7 @@ CSVFeedSpider
.. attribute:: headers
A list of the rows contained in the file CSV feed which will be used to
extract fields from it.
A list of the column names in the CSV file.
.. method:: parse_row(response, row)
@ -749,8 +751,8 @@ Combine SitemapSpider with other sources of urls::
def parse_other(self, response):
pass # ... scrape other here ...
.. _Sitemaps: http://www.sitemaps.org
.. _Sitemap index files: http://www.sitemaps.org/protocol.html#index
.. _Sitemaps: https://www.sitemaps.org/index.html
.. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
.. _robots.txt: http://www.robotstxt.org/
.. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
.. _Scrapyd documentation: http://scrapyd.readthedocs.org/en/latest/
.. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

2
docs/topics/ubuntu.rst
View File

@ -37,5 +37,5 @@ To use the packages:
.. warning:: `python-scrapy` is a different package provided by official debian
repositories, it's very outdated and it isn't supported by Scrapy team.
.. _Scrapinghub: http://scrapinghub.com/
.. _Scrapinghub: https://scrapinghub.com/
.. _GitHub repo: https://github.com/scrapy/scrapy

2
docs/utils/linkfix.py
View File

@ -20,7 +20,7 @@ _filename = None
_contents = None
# A regex that matches standard linkcheck output lines
line_re = re.compile(ur'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
# Read lines from the linkcheck output file
try:

2
extras/coverage-report.sh
View File

@ -1,6 +1,6 @@
# Run tests, generate coverage report and open it on a browser
#
# Requires: coverage 3.3 or above from http://pypi.python.org/pypi/coverage
# Requires: coverage 3.3 or above from https://pypi.python.org/pypi/coverage
coverage run --branch $(which trial) --reporter=text tests
coverage html -i

2
extras/qpsclient.py
View File

@ -41,7 +41,7 @@ class QPSSpider(Spider):
slots = int(self.slots)
if slots > 1:
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)]
urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in range(slots)]
else:
urls = [url]

2
requirements-py3.txt
View File

@ -1,4 +1,4 @@
Twisted >= 15.5.0
Twisted >= 17.9.0
lxml>=3.2.4
pyOpenSSL>=0.13.1
cssselect>=0.9

2
scrapy/VERSION
View File

@ -1 +1 @@
1.3.2
1.5.0

4
scrapy/__main__.py Normal file
View File

@ -0,0 +1,4 @@
from scrapy.cmdline import execute
if __name__ == '__main__':
execute()

4
scrapy/_monkeypatches.py
View File

@ -4,12 +4,12 @@ from six.moves import copyreg
if sys.version_info[0] == 2:
from urlparse import urlparse
# workaround for http://bugs.python.org/issue7904 - Python < 2.7
# workaround for https://bugs.python.org/issue7904 - Python < 2.7
if urlparse('s3://bucket/key').netloc != 'bucket':
from urlparse import uses_netloc
uses_netloc.append('s3')
# workaround for http://bugs.python.org/issue9374 - Python < 2.7.4
# workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
if urlparse('s3://bucket/key?key=value').query != 'key=value':
from urlparse import uses_query
uses_query.append('s3')

19
scrapy/cmdline.py
View File

@ -1,5 +1,5 @@
from __future__ import print_function
import sys
import sys, os
import optparse
import cProfile
import inspect
@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules
from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
from scrapy.settings.deprecated import check_deprecated_settings
def _iter_command_classes(module_name):
@ -20,7 +21,8 @@ def _iter_command_classes(module_name):
for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, ScrapyCommand) and \
obj.__module__ == module.__name__:
obj.__module__ == module.__name__ and \
not obj == ScrapyCommand:
yield obj
def _get_commands_from_module(module, inproject):
@ -106,6 +108,12 @@ def execute(argv=None, settings=None):
if settings is None:
settings = get_project_settings()
# set EDITOR from environment if available
try:
editor = os.environ['EDITOR']
except KeyError: pass
else:
settings['EDITOR'] = editor
check_deprecated_settings(settings)
# --- backwards compatibility for scrapy.conf.settings singleton ---
@ -158,4 +166,9 @@ def _run_command_profiled(cmd, args, opts):
p.dump_stats(opts.profile)
if __name__ == '__main__':
execute()
try:
execute()
finally:
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
# on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
garbage_collect()

7
scrapy/commands/edit.py
View File

@ -1,8 +1,10 @@
import sys, os
import sys
import os
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
class Command(ScrapyCommand):
requires_project = True
@ -15,7 +17,8 @@ class Command(ScrapyCommand):
return "Edit spider"
def long_desc(self):
return "Edit a spider using the editor defined in EDITOR setting"
return ("Edit a spider using the editor defined in the EDITOR environment"
" variable or else the EDITOR setting")
def _err(self, msg):
sys.stderr.write(msg + os.linesep)

30
scrapy/commands/parse.py
View File

@ -1,4 +1,5 @@
from __future__ import print_function
import json
import logging
from w3lib.url import is_url
@ -48,6 +49,8 @@ class Command(ScrapyCommand):
help="use CrawlSpider rules to discover the callback")
parser.add_option("-c", "--callback", dest="callback",
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
@ -142,7 +145,8 @@ class Command(ScrapyCommand):
logger.error('Unable to find spider for: %(url)s',
{'url': url})
request = Request(url, opts.callback)
# Request requires callback argument as callable or None, not string
request = Request(url, None)
_start_requests = lambda s: [self.prepare_request(s, request, opts)]
self.spidercls.start_requests = _start_requests
@ -164,7 +168,9 @@ class Command(ScrapyCommand):
# determine real callback
cb = response.meta['_callback']
if not cb:
if opts.rules and self.first_response == response:
if opts.callback:
cb = opts.callback
elif opts.rules and self.first_response == response:
cb = self.get_callback_from_rules(spider, response)
if not cb:
@ -201,6 +207,10 @@ class Command(ScrapyCommand):
req.callback = callback
return requests
#update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
request.meta.update(opts.meta)
request.meta['_depth'] = 1
request.meta['_callback'] = request.callback
request.callback = callback
@ -208,11 +218,27 @@ class Command(ScrapyCommand):
def process_options(self, args, opts):
ScrapyCommand.process_options(self, args, opts)
self.process_spider_arguments(opts)
self.process_request_meta(opts)
def process_spider_arguments(self, opts):
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_request_meta(self, opts):
if opts.meta:
try:
opts.meta = json.loads(opts.meta)
except ValueError:
raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
"Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
def run(self, args, opts):
# parse arguments
if not len(args) == 1 or not is_url(args[0]):

1
scrapy/commands/runspider.py
View File

@ -28,6 +28,7 @@ def _import_file(filepath):
class Command(ScrapyCommand):
requires_project = False
default_settings = {'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[options] <spider_file>"

3
scrapy/commands/settings.py
View File

@ -7,7 +7,8 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False}
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[options]"

5
scrapy/commands/startproject.py
View File

@ -26,7 +26,8 @@ IGNORE = ignore_patterns('*.pyc', '.svn')
class Command(ScrapyCommand):
requires_project = False
default_settings = {'LOG_ENABLED': False}
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "<project_name> [project_dir]"
@ -118,4 +119,4 @@ class Command(ScrapyCommand):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'project')

46
scrapy/commands/version.py
View File

@ -1,17 +1,14 @@
from __future__ import print_function
import sys
import platform
import twisted
import OpenSSL
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions
class Command(ScrapyCommand):
default_settings = {'LOG_ENABLED': False}
default_settings = {'LOG_ENABLED': False,
'SPIDER_LOADER_WARN_ONLY': True}
def syntax(self):
return "[-v]"
@ -26,38 +23,11 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if opts.verbose:
import cssselect
import parsel
import lxml.etree
import w3lib
lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
try:
w3lib_version = w3lib.__version__
except AttributeError:
w3lib_version = "<1.14.3"
print("Scrapy : %s" % scrapy.__version__)
print("lxml : %s" % lxml_version)
print("libxml2 : %s" % libxml2_version)
print("cssselect : %s" % cssselect.__version__)
print("parsel : %s" % parsel.__version__)
print("w3lib : %s" % w3lib_version)
print("Twisted : %s" % twisted.version.short())
print("Python : %s" % sys.version.replace("\n", "- "))
print("pyOpenSSL : %s" % self._get_openssl_version())
print("Platform : %s" % platform.platform())
versions = scrapy_components_versions()
width = max(len(n) for (n, _) in versions)
patt = "%-{}s : %s".format(width)
for name, version in versions:
print(patt % (name, version))
else:
print("Scrapy %s" % scrapy.__version__)
def _get_openssl_version(self):
try:
openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
.decode('ascii', errors='replace')
# pyOpenSSL 0.12 does not expose openssl version
except AttributeError:
openssl = 'Unknown OpenSSL version'
return '{} ({})'.format(OpenSSL.version.__version__, openssl)

4
scrapy/core/downloader/contextfactory.py
View File

@ -64,7 +64,7 @@ if twisted_version >= (14, 0, 0):
"""
Twisted-recommended context factory for web clients.
Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
"The default is to use a BrowserLikePolicyForHTTPS,
so unless you have special requirements you can leave this as-is."
@ -100,6 +100,6 @@ else:
def getContext(self, hostname=None, port=None):
ctx = ClientContextFactory.getContext(self)
# Enable all workarounds to SSL bugs as documented by
# http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
# https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html
ctx.set_options(SSL.OP_ALL)
return ctx

23
scrapy/core/downloader/handlers/datauri.py Normal file
View File

@ -0,0 +1,23 @@
from w3lib.url import parse_data_uri
from scrapy.http import TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
class DataURIDownloadHandler(object):
def __init__(self, settings):
super(DataURIDownloadHandler, self).__init__()
@defers
def download_request(self, request, spider):
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
if (issubclass(respcls, TextResponse) and
uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
return respcls(url=request.url, body=uri.data, **resp_kwargs)

73
scrapy/core/downloader/handlers/http11.py
View File

@ -15,6 +15,10 @@ from twisted.internet.error import TimeoutError
from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
HTTPConnectionPool, ResponseFailed
try:
from twisted.web.client import URI
except ImportError:
from twisted.web.client import _URI as URI
from twisted.internet.endpoints import TCP4ClientEndpoint
from scrapy.http import Headers
@ -228,10 +232,38 @@ class TunnelingAgent(Agent):
headers, bodyProducer, requestPath)
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI,
connectTimeout=None, bindAddress=None, pool=None):
super(ScrapyProxyAgent, self).__init__(reactor,
connectTimeout=connectTimeout,
bindAddress=bindAddress,
pool=pool)
self._proxyURI = URI.fromBytes(proxyURI)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
if twisted_version >= (15, 0, 0):
proxyEndpoint = self._getEndpoint(self._proxyURI)
else:
proxyEndpoint = self._getEndpoint(self._proxyURI.scheme,
self._proxyURI.host,
self._proxyURI.port)
key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
return self._requestWithEndpoint(key, proxyEndpoint, method,
URI.fromBytes(uri), headers,
bodyProducer, uri)
class ScrapyAgent(object):
_Agent = Agent
_ProxyAgent = ProxyAgent
_ProxyAgent = ScrapyProxyAgent
_TunnelingAgent = TunnelingAgent
def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,
@ -260,9 +292,8 @@ class ScrapyAgent(object):
contextFactory=self._contextFactory, connectTimeout=timeout,
bindAddress=bindaddress, pool=self._pool)
else:
endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort,
timeout=timeout, bindAddress=bindaddress)
return self._ProxyAgent(endpoint)
return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'),
connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
return self._Agent(reactor, contextFactory=self._contextFactory,
connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
@ -279,8 +310,7 @@ class ScrapyAgent(object):
headers.removeHeader(b'Proxy-Authorization')
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
else:
bodyproducer = None
elif method == b'POST':
# Setting Content-Length: 0 even for POST requests is not a
# MUST per HTTP RFCs, but it's common behavior, and some
# servers require this, otherwise returning HTTP 411 Length required
@ -289,10 +319,13 @@ class ScrapyAgent(object):
# "a Content-Length header field is normally sent in a POST
# request even when the value is 0 (indicating an empty payload body)."
#
# Twisted Agent will not add "Content-Length: 0" by itself
if method == b'POST':
headers.addRawHeader(b'Content-Length', b'0')
# Twisted < 17 will not add "Content-Length: 0" by itself;
# Twisted >= 17 fixes this;
# Using a producer with an empty-string sends `0` as Content-Length
# for all versions of Twisted.
bodyproducer = _RequestBodyProducer(b'')
else:
bodyproducer = None
start_time = time()
d = agent.request(
method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
@ -342,11 +375,12 @@ class ScrapyAgent(object):
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
"download warn size (%(warnsize)s).",
{'size': expected_size, 'warnsize': warnsize})
"download warn size (%(warnsize)s) in request %(request)s.",
{'size': expected_size, 'warnsize': warnsize, 'request': request})
def _cancel(_):
txresponse._transport._producer.loseConnection()
# Abort connection inmediately.
txresponse._transport._producer.abortConnection()
d = defer.Deferred(_cancel)
txresponse.deliverBody(_ResponseReader(
@ -399,14 +433,23 @@ class _ResponseReader(protocol.Protocol):
self._bytes_received = 0
def dataReceived(self, bodyBytes):
# This maybe called several times after cancel was called with buffered
# data.
if self._finished.called:
return
self._bodybuf.write(bodyBytes)
self._bytes_received += len(bodyBytes)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s).",
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize})
'maxsize': self._maxsize,
'request': self._request})
# Clear buffer earlier to avoid keeping data in memory for a long
# time.
self._bodybuf.truncate(0)
self._finished.cancel()
if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:

4
scrapy/core/engine.py
View File

@ -218,10 +218,8 @@ class ExecutionEngine(object):
request=request, spider=spider)
def download(self, request, spider):
slot = self.slot
slot.add_request(request)
d = self._download(request, spider)
d.addBoth(self._downloaded, slot, request, spider)
d.addBoth(self._downloaded, self.slot, request, spider)
return d
def _downloaded(self, response, slot, request, spider):

23
scrapy/crawler.py
View File

@ -11,12 +11,14 @@ from scrapy.core.engine import ExecutionEngine
from scrapy.resolver import CachingThreadedResolver
from scrapy.interfaces import ISpiderLoader
from scrapy.extension import ExtensionManager
from scrapy.settings import Settings
from scrapy.settings import overridden_settings, Settings
from scrapy.signalmanager import SignalManager
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.misc import load_object
from scrapy.utils.log import LogCounterHandler, configure_logging, log_scrapy_info
from scrapy.utils.log import (
LogCounterHandler, configure_logging, log_scrapy_info,
get_scrapy_root_handler, install_scrapy_root_handler)
from scrapy import signals
logger = logging.getLogger(__name__)
@ -32,11 +34,17 @@ class Crawler(object):
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
d = dict(overridden_settings(self.settings))
logger.info("Overridden settings: %(settings)r", {'settings': d})
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=settings.get('LOG_LEVEL'))
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
# lambda is assigned to Crawler attribute because this way it is not
# garbage collected after leaving __init__ scope
self.__remove_handler = lambda: logging.root.removeHandler(handler)
@ -75,7 +83,7 @@ class Crawler(object):
yield defer.maybeDeferred(self.engine.start)
except Exception:
# In Python 2 reraising an exception after yield discards
# the original traceback (see http://bugs.python.org/issue7563),
# the original traceback (see https://bugs.python.org/issue7563),
# so sys.exc_info() workaround is used.
# This workaround also works in Python 3, but it is not needed,
# and it is slower, so in Python 3 we use native `raise`.
@ -229,15 +237,18 @@ class CrawlerProcess(CrawlerRunner):
The CrawlerProcess object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
:param install_root_handler: whether to install root logging handler
(default: True)
This class shouldn't be needed (since Scrapy is responsible of using it
accordingly) unless writing scripts that manually handle the crawling
process. See :ref:`run-from-script` for an example.
"""
def __init__(self, settings=None):
def __init__(self, settings=None, install_root_handler=True):
super(CrawlerProcess, self).__init__(settings)
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)
def _signal_shutdown(self, signum, _):

2
scrapy/downloadermiddlewares/chunked.py
View File

@ -11,7 +11,7 @@ warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, "
class ChunkedTransferMiddleware(object):
"""This middleware adds support for chunked transfer encoding, as
documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
"""
def process_response(self, request, response, spider):

2
scrapy/downloadermiddlewares/httpcache.py
View File

@ -75,7 +75,7 @@ class HttpCacheMiddleware(object):
return response
# RFC2616 requires origin server to set Date header,
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
if 'Date' not in response.headers:
response.headers['Date'] = formatdate(usegmt=1)

6
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -1,6 +1,6 @@
import zlib
from scrapy.utils.gz import gunzip, is_gzipped
from scrapy.utils.gz import gunzip
from scrapy.http import Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.exceptions import NotConfigured
@ -34,11 +34,11 @@ class HttpCompressionMiddleware(object):
return response
if isinstance(response, Response):
content_encoding = response.headers.getlist('Content-Encoding')
if content_encoding and not is_gzipped(response):
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
respcls = responsetypes.from_args(headers=response.headers, \
url=response.url)
url=response.url, body=decoded_body)
kwargs = dict(cls=respcls, body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the

4
scrapy/downloadermiddlewares/redirect.py
View File

@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
request.meta.get('handle_httpstatus_all', False)):
return response
allowed_status = (301, 302, 303, 307)
allowed_status = (301, 302, 303, 307, 308)
if 'Location' not in response.headers or response.status not in allowed_status:
return response
@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
redirected_url = urljoin(request.url, location)
if response.status in (301, 307) or request.method == 'HEAD':
if response.status in (301, 307, 308) or request.method == 'HEAD':
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)

7
scrapy/downloadermiddlewares/retry.py
View File

@ -63,8 +63,13 @@ class RetryMiddleware(object):
def _retry(self, request, reason, spider):
retries = request.meta.get('retry_times', 0) + 1
retry_times = self.max_retry_times
if 'max_retry_times' in request.meta:
retry_times = request.meta['max_retry_times']
stats = spider.crawler.stats
if retries <= self.max_retry_times:
if retries <= retry_times:
logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
{'request': request, 'retries': retries, 'reason': reason},
extra={'spider': spider})

48
scrapy/exporters.py
View File

@ -36,6 +36,7 @@ class BaseItemExporter(object):
self.encoding = options.pop('encoding', None)
self.fields_to_export = options.pop('fields_to_export', None)
self.export_empty_fields = options.pop('export_empty_fields', False)
self.indent = options.pop('indent', None)
if not dont_fail and options:
raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))
@ -98,21 +99,33 @@ class JsonItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
self._configure(kwargs, dont_fail=True)
self.file = file
# there is a small difference between the behaviour or JsonItemExporter.indent
# and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
# the addition of newlines everywhere
json_indent = self.indent if self.indent is not None and self.indent > 0 else None
kwargs.setdefault('indent', json_indent)
kwargs.setdefault('ensure_ascii', not self.encoding)
self.encoder = ScrapyJSONEncoder(**kwargs)
self.first_item = True
def _beautify_newline(self):
if self.indent is not None:
self.file.write(b'\n')
def start_exporting(self):
self.file.write(b"[\n")
self.file.write(b"[")
self._beautify_newline()
def finish_exporting(self):
self.file.write(b"\n]")
self._beautify_newline()
self.file.write(b"]")
def export_item(self, item):
if self.first_item:
self.first_item = False
else:
self.file.write(b',\n')
self.file.write(b',')
self._beautify_newline()
itemdict = dict(self._get_serialized_fields(item))
data = self.encoder.encode(itemdict)
self.file.write(to_bytes(data, self.encoding))
@ -128,35 +141,54 @@ class XmlItemExporter(BaseItemExporter):
self.encoding = 'utf-8'
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self, new_item=False):
if self.indent is not None and (self.indent > 0 or new_item):
self._xg_characters('\n')
def _beautify_indent(self, depth=1):
if self.indent:
self._xg_characters(' ' * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()
self.xg.startElement(self.root_element, {})
self._beautify_newline(new_item=True)
def export_item(self, item):
self._beautify_indent(depth=1)
self.xg.startElement(self.item_element, {})
self._beautify_newline()
for name, value in self._get_serialized_fields(item, default_value=''):
self._export_xml_field(name, value)
self._export_xml_field(name, value, depth=2)
self._beautify_indent(depth=1)
self.xg.endElement(self.item_element)
self._beautify_newline(new_item=True)
def finish_exporting(self):
self.xg.endElement(self.root_element)
self.xg.endDocument()
def _export_xml_field(self, name, serialized_value):
def _export_xml_field(self, name, serialized_value, depth):
self._beautify_indent(depth=depth)
self.xg.startElement(name, {})
if hasattr(serialized_value, 'items'):
self._beautify_newline()
for subname, value in serialized_value.items():
self._export_xml_field(subname, value)
self._export_xml_field(subname, value, depth=depth+1)
self._beautify_indent(depth=depth)
elif is_listlike(serialized_value):
self._beautify_newline()
for value in serialized_value:
self._export_xml_field('value', value)
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
elif isinstance(serialized_value, six.text_type):
self._xg_characters(serialized_value)
else:
self._xg_characters(str(serialized_value))
self.xg.endElement(name)
self._beautify_newline()
# Workaround for http://bugs.python.org/issue17606
# Workaround for https://bugs.python.org/issue17606
# Before Python 2.7.4 xml.sax.saxutils required bytes;
# since 2.7.4 it requires unicode. The bug is likely to be
# fixed in 2.7.6, but 2.7.6 will still support unicode,

5
scrapy/extensions/feedexport.py
View File

@ -172,6 +172,9 @@ class FeedExporter(object):
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
self._exporting = False
self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
self.indent = None
if settings.get('FEED_EXPORT_INDENT') is not None:
self.indent = settings.getint('FEED_EXPORT_INDENT')
uripar = settings['FEED_URI_PARAMS']
self._uripar = load_object(uripar) if uripar else lambda x, y: None
@ -188,7 +191,7 @@ class FeedExporter(object):
storage = self._get_storage(uri)
file = storage.open(spider)
exporter = self._get_exporter(file, fields_to_export=self.export_fields,
encoding=self.export_encoding)
encoding=self.export_encoding, indent=self.indent)
if self.store_empty:
exporter.start_exporting()
self._exporting = True

24
scrapy/extensions/httpcache.py
View File

@ -1,6 +1,7 @@
from __future__ import print_function
import os
import gzip
import logging
from six.moves import cPickle as pickle
from importlib import import_module
from time import time
@ -12,7 +13,10 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.request import request_fingerprint
from scrapy.utils.project import data_path
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
logger = logging.getLogger(__name__)
class DummyPolicy(object):
@ -66,8 +70,8 @@ class RFC2616Policy(object):
return True
def should_cache_response(self, response, request):
# What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
# Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
# What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
# Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
# Status code 206 is not included because cache can not deal with partial contents
cc = self._parse_cachecontrol(response)
# obey directive "Cache-Control: no-store"
@ -159,7 +163,7 @@ class RFC2616Policy(object):
def _compute_freshness_lifetime(self, response, request, now):
# Reference nsHttpResponseHead::ComputeFreshnessLifetime
# http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#410
# https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706
cc = self._parse_cachecontrol(response)
maxage = self._get_max_age(cc)
if maxage is not None:
@ -190,7 +194,7 @@ class RFC2616Policy(object):
def _compute_current_age(self, response, request, now):
# Reference nsHttpResponseHead::ComputeCurrentAge
# http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#366
# https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658
currentage = 0
# If Date header is not set we assume it is a fast connection, and
# clock is in sync with the server
@ -220,6 +224,8 @@ class DbmCacheStorage(object):
dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
self.db = self.dbmodule.open(dbpath, 'c')
logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
def close_spider(self, spider):
self.db.close()
@ -272,7 +278,8 @@ class FilesystemCacheStorage(object):
self._open = gzip.open if self.use_gzip else open
def open_spider(self, spider):
pass
logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir},
extra={'spider': spider})
def close_spider(self, spider):
pass
@ -348,11 +355,14 @@ class LeveldbCacheStorage(object):
dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
self.db = self._leveldb.LevelDB(dbpath)
logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
def close_spider(self, spider):
# Do compactation each time to save space and also recreate files to
# avoid them being removed in storages with timestamp-based autoremoval.
self.db.CompactRange()
del self.db
garbage_collect()
def retrieve_response(self, spider, request):
data = self._read_data(spider, request)
@ -404,7 +414,7 @@ class LeveldbCacheStorage(object):
def parse_cachecontrol(header):
"""Parse Cache-Control header
http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
>>> parse_cachecontrol(b'public, max-age=3600') == {b'public': None,
... b'max-age': b'3600'}

1
scrapy/extensions/memusage.py
View File

@ -35,7 +35,6 @@ class MemoryUsage(object):
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
self.report = crawler.settings.getbool('MEMUSAGE_REPORT')
self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
self.mail = MailSender.from_settings(crawler.settings)
crawler.signals.connect(self.engine_started, signal=signals.engine_started)

2
scrapy/extensions/telnet.py
View File

@ -82,7 +82,7 @@ class TelnetConsole(protocol.ServerFactory):
'prefs': print_live_refs,
'hpy': hpy,
'help': "This is Scrapy telnet console. For more info see: " \
"http://doc.scrapy.org/en/latest/topics/telnetconsole.html",
"https://doc.scrapy.org/en/latest/topics/telnetconsole.html",
}
self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
return telnet_vars

4
scrapy/http/request/__init__.py
View File

@ -27,6 +27,10 @@ class Request(object_ref):
assert isinstance(priority, int), "Request priority not an integer: %r" % priority
self.priority = priority
if callback is not None and not callable(callback):
raise TypeError('callback must be a callable, got %s' % type(callback).__name__)
if errback is not None and not callable(errback):
raise TypeError('errback must be a callable, got %s' % type(errback).__name__)
assert callback or not errback, "Cannot use errback without a callback"
self.callback = callback
self.errback = errback

2
scrapy/http/request/form.py
View File

@ -135,7 +135,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
if clickable and clickable[0] not in formdata and not clickable[0] is None:
values.append(clickable)
values.extend(formdata.items())
values.extend((k, v) for k, v in formdata.items() if v is not None)
return values

9
scrapy/http/response/text.py
View File

@ -135,7 +135,7 @@ class TextResponse(Response):
* an attribute Selector (not SelectorList) - e.g.
``response.css('a::attr(href)')[0]`` or
``response.xpath('//img/@src')[0]``.
* a Selector for ``<a>`` element, e.g.
* a Selector for ``<a>`` or ``<link>`` element, e.g.
``response.css('a.my_link')[0]``.
See :ref:`response-follow-example` for usage examples.
@ -165,10 +165,11 @@ def _url_from_selector(sel):
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
raise ValueError("Unsupported selector: %s" % sel)
if sel.root.tag != 'a':
raise ValueError("Only <a> elements are supported; got <%s>" %
if sel.root.tag not in ('a', 'link'):
raise ValueError("Only <a> and <link> elements are supported; got <%s>" %
sel.root.tag)
href = sel.root.get('href')
if href is None:
raise ValueError("<a> element has no href attribute: %s" % sel)
raise ValueError("<%s> element has no href attribute: %s" %
(sel.root.tag, sel))
return strip_html5_whitespace(href)

4
scrapy/linkextractors/__init__.py
View File

@ -28,7 +28,7 @@ IGNORED_EXTENSIONS = [
# video
'3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
'm4a',
'm4a', 'm4v',
# office suites
'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',
@ -101,7 +101,7 @@ class FilteringLinkExtractor(object):
links = [x for x in links if self._link_allowed(x)]
if self.canonicalize:
for link in links:
link.url = canonicalize_url(urlparse(link.url))
link.url = canonicalize_url(link.url)
links = self.link_extractor._process_links(links)
return links

22
scrapy/linkextractors/lxmlhtml.py
View File

@ -6,6 +6,7 @@ from six.moves.urllib.parse import urljoin
import lxml.etree as etree
from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@ -29,12 +30,17 @@ def _nons(tag):
class LxmlParserLinkExtractor(object):
def __init__(self, tag="a", attr="href", process=None, unique=False,
strip=True):
strip=True, canonicalized=False):
self.scan_tag = tag if callable(tag) else lambda t: t == tag
self.scan_attr = attr if callable(attr) else lambda a: a == attr
self.process_attr = process if callable(process) else lambda v: v
self.unique = unique
self.strip = strip
if canonicalized:
self.link_key = lambda link: link.url
else:
self.link_key = lambda link: canonicalize_url(link.url,
keep_fragments=True)
def _iter_links(self, document):
for el in document.iter(etree.Element):
@ -82,21 +88,27 @@ class LxmlParserLinkExtractor(object):
def _deduplicate_if_needed(self, links):
if self.unique:
return unique_list(links, key=lambda link: link.url)
return unique_list(links, key=self.link_key)
return links
class LxmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=True,
tags=('a', 'area'), attrs=('href',), canonicalize=False,
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
strip=True):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
unique=unique, process=process_value, strip=strip)
lx = LxmlParserLinkExtractor(
tag=tag_func,
attr=attr_func,
unique=unique,
process=process_value,
strip=strip,
canonicalized=canonicalize
)
super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
allow_domains=allow_domains, deny_domains=deny_domains,

18
scrapy/linkextractors/sgml.py
View File

@ -6,7 +6,7 @@ from six.moves.urllib.parse import urljoin
import warnings
from sgmllib import SGMLParser
from w3lib.url import safe_url_string
from w3lib.url import safe_url_string, canonicalize_url
from w3lib.html import strip_html5_whitespace
from scrapy.link import Link
@ -20,7 +20,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
class BaseSgmlLinkExtractor(SGMLParser):
def __init__(self, tag="a", attr="href", unique=False, process_value=None,
strip=True):
strip=True, canonicalized=False):
warnings.warn(
"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
"Please use scrapy.linkextractors.LinkExtractor",
@ -33,6 +33,11 @@ class BaseSgmlLinkExtractor(SGMLParser):
self.current_link = None
self.unique = unique
self.strip = strip
if canonicalized:
self.link_key = lambda link: link.url
else:
self.link_key = lambda link: canonicalize_url(link.url,
keep_fragments=True)
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
""" Do the real extraction work """
@ -61,8 +66,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
The subclass should override it if necessary
"""
links = unique_list(links, key=lambda link: link.url) if self.unique else links
return links
return unique_list(links, key=self.link_key) if self.unique else links
def extract_links(self, response):
# wrapper needed to allow to work directly with text
@ -107,10 +111,9 @@ class BaseSgmlLinkExtractor(SGMLParser):
class SgmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
process_value=None, deny_extensions=None, restrict_css=(),
strip=True):
warnings.warn(
"SgmlLinkExtractor is deprecated and will be removed in future releases. "
"Please use scrapy.linkextractors.LinkExtractor",
@ -124,7 +127,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
with warnings.catch_warnings():
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
unique=unique, process_value=process_value, strip=strip)
unique=unique, process_value=process_value, strip=strip,
canonicalized=canonicalize)
super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
allow_domains=allow_domains, deny_domains=deny_domains,

2
scrapy/logformatter.py
View File

@ -43,6 +43,8 @@ class LogFormatter(object):
'request_flags' : request_flags,
'referer': referer_str(request),
'response_flags': response_flags,
# backward compatibility with Scrapy logformatter below 1.4 version
'flags': response_flags
}
}

22
scrapy/mail.py
View File

@ -5,7 +5,10 @@ See documentation in docs/topics/email.rst
"""
import logging
from six.moves import cStringIO as StringIO
try:
from cStringIO import StringIO as BytesIO
except ImportError:
from io import BytesIO
import six
from email.utils import COMMASPACE, formatdate
@ -21,19 +24,26 @@ else:
from twisted.internet import defer, reactor, ssl
from .utils.misc import arg_to_iter
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
logger = logging.getLogger(__name__)
def _to_bytes_or_none(text):
if text is None:
return None
return to_bytes(text)
class MailSender(object):
def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
self.smtphost = smtphost
self.smtpport = smtpport
self.smtpuser = smtpuser
self.smtppass = smtppass
self.smtpuser = _to_bytes_or_none(smtpuser)
self.smtppass = _to_bytes_or_none(smtppass)
self.smtptls = smtptls
self.smtpssl = smtpssl
self.mailfrom = mailfrom
@ -88,7 +98,7 @@ class MailSender(object):
'mailattachs': len(attachs)})
return
dfd = self._sendmail(rcpts, msg.as_string())
dfd = self._sendmail(rcpts, msg.as_string().encode(charset or 'utf-8'))
dfd.addCallbacks(self._sent_ok, self._sent_failed,
callbackArgs=[to, cc, subject, len(attachs)],
errbackArgs=[to, cc, subject, len(attachs)])
@ -112,7 +122,7 @@ class MailSender(object):
def _sendmail(self, to_addrs, msg):
# Import twisted.mail here because it is not available in python3
from twisted.mail.smtp import ESMTPSenderFactory
msg = StringIO(msg)
msg = BytesIO(msg)
d = defer.Deferred()
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \

51
scrapy/pipelines/files.py
View File

@ -57,7 +57,7 @@ class FSFilesStore(object):
absolute_path = self._get_filesystem_path(path)
try:
last_modified = os.path.getmtime(absolute_path)
except: # FIXME: catching everything!
except os.error:
return {}
with open(absolute_path, 'rb') as f:
@ -128,7 +128,7 @@ class S3FilesStore(object):
def _get_boto_bucket(self):
# disable ssl (is_secure=False) because of this python bug:
# http://bugs.python.org/issue5103
# https://bugs.python.org/issue5103
c = self.S3Connection(self.AWS_ACCESS_KEY_ID, self.AWS_SECRET_ACCESS_KEY, is_secure=False)
return c.get_bucket(self.bucket, validate=False)
@ -202,6 +202,47 @@ class S3FilesStore(object):
return extra
class GCSFilesStore(object):
GCS_PROJECT_ID = None
CACHE_CONTROL = 'max-age=172800'
def __init__(self, uri):
from google.cloud import storage
client = storage.Client(project=self.GCS_PROJECT_ID)
bucket, prefix = uri[5:].split('/', 1)
self.bucket = client.bucket(bucket)
self.prefix = prefix
def stat_file(self, path, info):
def _onsuccess(blob):
if blob:
checksum = blob.md5_hash
last_modified = time.mktime(blob.updated.timetuple())
return {'checksum': checksum, 'last_modified': last_modified}
else:
return {}
return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
def _get_content_type(self, headers):
if headers and 'Content-Type' in headers:
return headers['Content-Type']
else:
return 'application/octet-stream'
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
content_type=self._get_content_type(headers)
)
class FilesPipeline(MediaPipeline):
"""Abstract pipeline that implement the file downloading
@ -227,6 +268,7 @@ class FilesPipeline(MediaPipeline):
'': FSFilesStore,
'file': FSFilesStore,
's3': S3FilesStore,
'gs': GCSFilesStore,
}
DEFAULT_FILES_URLS_FIELD = 'file_urls'
DEFAULT_FILES_RESULT_FIELD = 'files'
@ -257,7 +299,7 @@ class FilesPipeline(MediaPipeline):
resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
)
super(FilesPipeline, self).__init__(download_func=download_func)
super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
@classmethod
def from_settings(cls, settings):
@ -270,6 +312,9 @@ class FilesPipeline(MediaPipeline):
s3store.AWS_VERIFY = settings['AWS_VERIFY']
s3store.POLICY = settings['FILES_STORE_S3_ACL']
gcs_store = cls.STORE_SCHEMES['gs']
gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)

8
scrapy/pipelines/images.py
View File

@ -91,6 +91,9 @@ class ImagesPipeline(FilesPipeline):
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
gcs_store = cls.STORE_SCHEMES['gs']
gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
store_uri = settings['IMAGES_STORE']
return cls(store_uri, settings=settings)
@ -132,6 +135,11 @@ class ImagesPipeline(FilesPipeline):
background = Image.new('RGBA', image.size, (255, 255, 255))
background.paste(image, image)
image = background.convert('RGB')
elif image.mode == 'P':
image = image.convert("RGBA")
background = Image.new('RGBA', image.size, (255, 255, 255))
background.paste(image, image)
image = background.convert('RGB')
elif image.mode != 'RGB':
image = image.convert('RGB')

27
scrapy/pipelines/media.py
View File

@ -1,10 +1,13 @@
from __future__ import print_function
import functools
import logging
from collections import defaultdict
from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure
from scrapy.settings import Settings
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import arg_to_iter
@ -24,9 +27,23 @@ class MediaPipeline(object):
self.downloaded = {}
self.waiting = defaultdict(list)
def __init__(self, download_func=None):
def __init__(self, download_func=None, settings=None):
self.download_func = download_func
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
resolve = functools.partial(self._key_for_pipe,
base_class_name="MediaPipeline",
settings=settings)
self.allow_redirects = settings.getbool(
resolve('MEDIA_ALLOW_REDIRECTS'), False
)
self._handle_statuses(self.allow_redirects)
def _handle_statuses(self, allow_redirects):
self.handle_httpstatus_list = None
if allow_redirects:
self.handle_httpstatus_list = SequenceExclude(range(300, 400))
def _key_for_pipe(self, key, base_class_name=None,
settings=None):
@ -93,6 +110,12 @@ class MediaPipeline(object):
)
return dfd.addBoth(lambda _: wad) # it must return wad at last
def _modify_media_request(self, request):
if self.handle_httpstatus_list:
request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
else:
request.meta['handle_httpstatus_all'] = True
def _check_media_to_download(self, result, request, info):
if result is not None:
return result
@ -103,7 +126,7 @@ class MediaPipeline(object):
callback=self.media_downloaded, callbackArgs=(request, info),
errback=self.media_failed, errbackArgs=(request, info))
else:
request.meta['handle_httpstatus_all'] = True
self._modify_media_request(request)
dfd = self.crawler.engine.download(request, info.spider)
dfd.addCallbacks(
callback=self.media_downloaded, callbackArgs=(request, info),

3
scrapy/resolver.py
View File

@ -22,7 +22,8 @@ class CachingThreadedResolver(ThreadedResolver):
# to enforce Scrapy's DNS_TIMEOUT setting's value
timeout = (self.timeout,)
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
d.addCallback(self._cache_result, name)
if dnscache.limit:
d.addCallback(self._cache_result, name)
return d
def _cache_result(self, result, name):

20
scrapy/settings/default_settings.py
View File

@ -13,7 +13,6 @@ Scrapy developers, if you add a setting here remember to:
"""
import os
import sys
from importlib import import_module
from os.path import join, abspath, dirname
@ -67,6 +66,7 @@ DOWNLOAD_DELAY = 0
DOWNLOAD_HANDLERS = {}
DOWNLOAD_HANDLERS_BASE = {
'data': 'scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler',
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
@ -113,13 +113,9 @@ DOWNLOADER_STATS = True
DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'
try:
EDITOR = os.environ['EDITOR']
except KeyError:
if sys.platform == 'win32':
EDITOR = '%s -m idlelib.idle'
else:
EDITOR = 'vi'
EDITOR = 'vi'
if sys.platform == 'win32':
EDITOR = '%s -m idlelib.idle'
EXTENSIONS = {}
@ -160,6 +156,7 @@ FEED_EXPORTERS_BASE = {
'marshal': 'scrapy.exporters.MarshalItemExporter',
'pickle': 'scrapy.exporters.PickleItemExporter',
}
FEED_EXPORT_INDENT = 0
FILES_STORE_S3_ACL = 'private'
@ -217,7 +214,6 @@ MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
MEMUSAGE_ENABLED = True
MEMUSAGE_LIMIT_MB = 0
MEMUSAGE_NOTIFY_MAIL = []
MEMUSAGE_REPORT = False
MEMUSAGE_WARNING_MB = 0
METAREFRESH_ENABLED = True
@ -234,10 +230,11 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
RETRY_HTTP_CODES = [500, 502, 503, 504, 408]
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
RETRY_PRIORITY_ADJUST = -1
ROBOTSTXT_OBEY = False
@ -248,6 +245,7 @@ SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
SPIDER_LOADER_WARN_ONLY = False
SPIDER_MIDDLEWARES = {}
@ -272,7 +270,7 @@ TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
URLLENGTH_LIMIT = 2083
USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__
USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__version__
TELNETCONSOLE_ENABLED = 1
TELNETCONSOLE_PORT = [6023, 6073]

2
scrapy/shell.py
View File

@ -164,7 +164,7 @@ class Shell(object):
def inspect_response(response, spider):
"""Open a shell to inspect the given response"""
Shell(spider.crawler).start(response=response)
Shell(spider.crawler).start(response=response, spider=spider)
def _request_deferred(request):

2
scrapy/signalmanager.py
View File

@ -55,7 +55,7 @@ class SignalManager(object):
The keyword arguments are passed to the signal handlers (connected
through the :meth:`connect` method).
.. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html
.. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
"""
kwargs.setdefault('sender', self.sender)
return _signal.send_catch_log_deferred(signal, **kwargs)

28
scrapy/spiderloader.py
View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from collections import defaultdict
import traceback
import warnings
@ -18,11 +19,26 @@ class SpiderLoader(object):
"""
def __init__(self, settings):
self.spider_modules = settings.getlist('SPIDER_MODULES')
self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
self._spiders = {}
self._found = defaultdict(list)
self._load_all_spiders()
def _check_name_duplicates(self):
dupes = ["\n".join(" {cls} named {name!r} (in {module})".format(
module=mod, cls=cls, name=name)
for (mod, cls) in locations)
for name, locations in self._found.items()
if len(locations)>1]
if dupes:
msg = ("There are several spiders with the same name:\n\n"
"{}\n\n This can cause unexpected behavior.".format(
"\n\n".join(dupes)))
warnings.warn(msg, UserWarning)
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
self._found[spcls.name].append((module.__name__, spcls.__name__))
self._spiders[spcls.name] = spcls
def _load_all_spiders(self):
@ -31,10 +47,14 @@ class SpiderLoader(object):
for module in walk_modules(name):
self._load_spiders(module)
except ImportError as e:
msg = ("\n{tb}Could not load spiders from module '{modname}'. "
"Check SPIDER_MODULES setting".format(
modname=name, tb=traceback.format_exc()))
warnings.warn(msg, RuntimeWarning)
if self.warn_only:
msg = ("\n{tb}Could not load spiders from module '{modname}'. "
"See above traceback for details.".format(
modname=name, tb=traceback.format_exc()))
warnings.warn(msg, RuntimeWarning)
else:
raise
self._check_name_duplicates()
@classmethod
def from_settings(cls, settings):

10
scrapy/spidermiddlewares/offsite.py
View File

@ -6,6 +6,7 @@ See documentation in docs/topics/spider-middleware.rst
import re
import logging
import warnings
from scrapy import signals
from scrapy.http import Request
@ -52,9 +53,18 @@ class OffsiteMiddleware(object):
allowed_domains = getattr(spider, 'allowed_domains', None)
if not allowed_domains:
return re.compile('') # allow all by default
url_pattern = re.compile("^https?://.*$")
for domain in allowed_domains:
if url_pattern.match(domain):
warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning)
regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
return re.compile(regex)
def spider_opened(self, spider):
self.host_regex = self.get_host_regex(spider)
self.domains_seen = set()
class URLWarning(Warning):
pass

343
scrapy/spidermiddlewares/referer.py
View File

@ -2,22 +2,359 @@
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
from six.moves.urllib.parse import urlparse
import warnings
from scrapy.http import Request
from w3lib.url import safe_url_string
from scrapy.http import Request, Response
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.utils.python import to_native_str
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.url import strip_url
LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
POLICY_NO_REFERRER = "no-referrer"
POLICY_NO_REFERRER_WHEN_DOWNGRADE = "no-referrer-when-downgrade"
POLICY_SAME_ORIGIN = "same-origin"
POLICY_ORIGIN = "origin"
POLICY_STRICT_ORIGIN = "strict-origin"
POLICY_ORIGIN_WHEN_CROSS_ORIGIN = "origin-when-cross-origin"
POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN = "strict-origin-when-cross-origin"
POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"
class ReferrerPolicy(object):
NOREFERRER_SCHEMES = LOCAL_SCHEMES
def referrer(self, response_url, request_url):
raise NotImplementedError()
def stripped_referrer(self, url):
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
return self.strip_url(url)
def origin_referrer(self, url):
if urlparse(url).scheme not in self.NOREFERRER_SCHEMES:
return self.origin(url)
def strip_url(self, url, origin_only=False):
"""
https://www.w3.org/TR/referrer-policy/#strip-url
If url is null, return no referrer.
If url's scheme is a local scheme, then return no referrer.
Set url's username to the empty string.
Set url's password to null.
Set url's fragment to null.
If the origin-only flag is true, then:
Set url's path to null.
Set url's query to null.
Return url.
"""
if not url:
return None
return strip_url(url,
strip_credentials=True,
strip_fragment=True,
strip_default_port=True,
origin_only=origin_only)
def origin(self, url):
"""Return serialized origin (scheme, host, path) for a request or response URL."""
return self.strip_url(url, origin_only=True)
def potentially_trustworthy(self, url):
# Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
parsed_url = urlparse(url)
if parsed_url.scheme in ('data',):
return False
return self.tls_protected(url)
def tls_protected(self, url):
return urlparse(url).scheme in ('https', 'ftps')
class NoReferrerPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer
The simplest policy is "no-referrer", which specifies that no referrer information
is to be sent along with requests made from a particular request client to any origin.
The header will be omitted entirely.
"""
name = POLICY_NO_REFERRER
def referrer(self, response_url, request_url):
return None
class NoReferrerWhenDowngradePolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade
The "no-referrer-when-downgrade" policy sends a full URL along with requests
from a TLS-protected environment settings object to a potentially trustworthy URL,
and requests from clients which are not TLS-protected to any origin.
Requests from TLS-protected clients to non-potentially trustworthy URLs,
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
This is a user agent's default behavior, if no policy is otherwise specified.
"""
name = POLICY_NO_REFERRER_WHEN_DOWNGRADE
def referrer(self, response_url, request_url):
if not self.tls_protected(response_url) or self.tls_protected(request_url):
return self.stripped_referrer(response_url)
class SameOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-same-origin
The "same-origin" policy specifies that a full URL, stripped for use as a referrer,
is sent as referrer information when making same-origin requests from a particular request client.
Cross-origin requests, on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name = POLICY_SAME_ORIGIN
def referrer(self, response_url, request_url):
if self.origin(response_url) == self.origin(request_url):
return self.stripped_referrer(response_url)
class OriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-origin
The "origin" policy specifies that only the ASCII serialization
of the origin of the request client is sent as referrer information
when making both same-origin requests and cross-origin requests
from a particular request client.
"""
name = POLICY_ORIGIN
def referrer(self, response_url, request_url):
return self.origin_referrer(response_url)
class StrictOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin
The "strict-origin" policy sends the ASCII serialization
of the origin of the request client when making requests:
- from a TLS-protected environment settings object to a potentially trustworthy URL, and
- from non-TLS-protected environment settings objects to any origin.
Requests from TLS-protected request clients to non- potentially trustworthy URLs,
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name = POLICY_STRICT_ORIGIN
def referrer(self, response_url, request_url):
if ((self.tls_protected(response_url) and
self.potentially_trustworthy(request_url))
or not self.tls_protected(response_url)):
return self.origin_referrer(response_url)
class OriginWhenCrossOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-origin-when-cross-origin
The "origin-when-cross-origin" policy specifies that a full URL,
stripped for use as a referrer, is sent as referrer information
when making same-origin requests from a particular request client,
and only the ASCII serialization of the origin of the request client
is sent as referrer information when making cross-origin requests
from a particular request client.
"""
name = POLICY_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
else:
return origin
class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-strict-origin-when-cross-origin
The "strict-origin-when-cross-origin" policy specifies that a full URL,
stripped for use as a referrer, is sent as referrer information
when making same-origin requests from a particular request client,
and only the ASCII serialization of the origin of the request client
when making cross-origin requests:
- from a TLS-protected environment settings object to a potentially trustworthy URL, and
- from non-TLS-protected environment settings objects to any origin.
Requests from TLS-protected clients to non- potentially trustworthy URLs,
on the other hand, will contain no referrer information.
A Referer HTTP header will not be sent.
"""
name = POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN
def referrer(self, response_url, request_url):
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
elif ((self.tls_protected(response_url) and
self.potentially_trustworthy(request_url))
or not self.tls_protected(response_url)):
return self.origin_referrer(response_url)
class UnsafeUrlPolicy(ReferrerPolicy):
"""
https://www.w3.org/TR/referrer-policy/#referrer-policy-unsafe-url
The "unsafe-url" policy specifies that a full URL, stripped for use as a referrer,
is sent along with both cross-origin requests
and same-origin requests made from a particular request client.
Note: The policy's name doesn't lie; it is unsafe.
This policy will leak origins and paths from TLS-protected resources
to insecure origins.
Carefully consider the impact of setting such a policy for potentially sensitive documents.
"""
name = POLICY_UNSAFE_URL
def referrer(self, response_url, request_url):
return self.stripped_referrer(response_url)
class DefaultReferrerPolicy(NoReferrerWhenDowngradePolicy):
"""
A variant of "no-referrer-when-downgrade",
with the addition that "Referer" is not sent if the parent request was
using ``file://`` or ``s3://`` scheme.
"""
NOREFERRER_SCHEMES = LOCAL_SCHEMES + ('file', 's3')
name = POLICY_SCRAPY_DEFAULT
_policy_classes = {p.name: p for p in (
NoReferrerPolicy,
NoReferrerWhenDowngradePolicy,
SameOriginPolicy,
OriginPolicy,
StrictOriginPolicy,
OriginWhenCrossOriginPolicy,
StrictOriginWhenCrossOriginPolicy,
UnsafeUrlPolicy,
DefaultReferrerPolicy,
)}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
_policy_classes[''] = NoReferrerWhenDowngradePolicy
def _load_policy_class(policy, warning_only=False):
"""
Expect a string for the path to the policy class,
otherwise try to interpret the string as a standard value
from https://www.w3.org/TR/referrer-policy/#referrer-policies
"""
try:
return load_object(policy)
except ValueError:
try:
return _policy_classes[policy.lower()]
except KeyError:
msg = "Could not load referrer policy %r" % policy
if not warning_only:
raise RuntimeError(msg)
else:
warnings.warn(msg, RuntimeWarning)
return None
class RefererMiddleware(object):
def __init__(self, settings=None):
self.default_policy = DefaultReferrerPolicy
if settings is not None:
self.default_policy = _load_policy_class(
settings.get('REFERRER_POLICY'))
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('REFERER_ENABLED'):
raise NotConfigured
return cls()
mw = cls(crawler.settings)
# Note: this hook is a bit of a hack to intercept redirections
crawler.signals.connect(mw.request_scheduled, signal=signals.request_scheduled)
return mw
def policy(self, resp_or_url, request):
"""
Determine Referrer-Policy to use from a parent Response (or URL),
and a Request to be sent.
- if a valid policy is set in Request meta, it is used.
- if the policy is set in meta but is wrong (e.g. a typo error),
the policy from settings is used
- if the policy is not set in Request meta,
but there is a Referrer-policy header in the parent response,
it is used if valid
- otherwise, the policy from settings is used.
"""
policy_name = request.meta.get('referrer_policy')
if policy_name is None:
if isinstance(resp_or_url, Response):
policy_header = resp_or_url.headers.get('Referrer-Policy')
if policy_header is not None:
policy_name = to_native_str(policy_header.decode('latin1'))
if policy_name is None:
return self.default_policy()
cls = _load_policy_class(policy_name, warning_only=True)
return cls() if cls else self.default_policy()
def process_spider_output(self, response, result, spider):
def _set_referer(r):
if isinstance(r, Request):
r.headers.setdefault('Referer', response.url)
referrer = self.policy(response, r).referrer(response.url, r.url)
if referrer is not None:
r.headers.setdefault('Referer', referrer)
return r
return (_set_referer(r) for r in result or ())
def request_scheduled(self, request, spider):
# check redirected request to patch "Referer" header if necessary
redirected_urls = request.meta.get('redirect_urls', [])
if redirected_urls:
request_referrer = request.headers.get('Referer')
# we don't patch the referrer value if there is none
if request_referrer is not None:
# the request's referrer header value acts as a surrogate
# for the parent response URL
#
# Note: if the 3xx response contained a Referrer-Policy header,
# the information is not available using this hook
parent_url = safe_url_string(request_referrer)
policy_referrer = self.policy(parent_url, request).referrer(
parent_url, request.url)
if policy_referrer != request_referrer:
if policy_referrer is None:
request.headers.pop('Referer')
else:
request.headers['Referer'] = policy_referrer

2
scrapy/spiders/__init__.py
View File

@ -87,7 +87,7 @@ class Spider(object_ref):
return Request(url, dont_filter=True)
def parse(self, response):
raise NotImplementedError
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
@classmethod
def update_settings(cls, settings):

20
scrapy/spiders/sitemap.py
View File

@ -5,7 +5,8 @@ import six
from scrapy.spiders import Spider
from scrapy.http import Request, XmlResponse
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
from scrapy.utils.gz import gunzip, is_gzipped
from scrapy.utils.gz import gunzip, gzip_magic_number
logger = logging.getLogger(__name__)
@ -47,7 +48,7 @@ class SitemapSpider(Spider):
if any(x.search(loc) for x in self._follow):
yield Request(loc, callback=self._parse_sitemap)
elif s.type == 'urlset':
for loc in iterloc(s):
for loc in iterloc(s, self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield Request(loc, callback=c)
@ -59,12 +60,19 @@ class SitemapSpider(Spider):
"""
if isinstance(response, XmlResponse):
return response.body
elif is_gzipped(response):
elif gzip_magic_number(response):
return gunzip(response.body)
elif response.url.endswith('.xml'):
# actual gzipped sitemap files are decompressed above ;
# if we are here (response body is not gzipped)
# and have a response for .xml.gz,
# it usually means that it was already gunzipped
# by HttpCompression middleware,
# the HTTP response being sent with "Content-Encoding: gzip"
# without actually being a .xml.gz file in the first place,
# merely XML gzip-compressed on the fly,
# in other word, here, we have plain XML
elif response.url.endswith('.xml') or response.url.endswith('.xml.gz'):
return response.body
elif response.url.endswith('.xml.gz'):
return gunzip(response.body)
def regex(x):

Some files were not shown because too many files have changed in this diff Show More