diff --git a/.bandit.yml b/.bandit.yml
new file mode 100644
index 000000000..243379b0b
--- /dev/null
+++ b/.bandit.yml
@@ -0,0 +1,18 @@
+skips:
+- B101
+- B105
+- B301
+- B303
+- B306
+- B307
+- B311
+- B320
+- B321
+- B402 # https://github.com/scrapy/scrapy/issues/4180
+- B403
+- B404
+- B406
+- B410
+- B503
+- B603
+- B605
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 6e7be142e..c9f1abea5 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 1.5.0
+current_version = 1.8.0
commit = True
tag = True
tag_name = {new_version}
diff --git a/.coveragerc b/.coveragerc
index 3105409ba..02acbff8e 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -3,17 +3,3 @@ branch = true
include = scrapy/*
omit =
tests/*
- scrapy/xlib/*
- scrapy/conf.py
- scrapy/stats.py
- scrapy/project.py
- scrapy/utils/decorator.py
- scrapy/statscol.py
- scrapy/squeue.py
- scrapy/log.py
- scrapy/dupefilter.py
- scrapy/command.py
- scrapy/linkextractor.py
- scrapy/spider.py
- scrapy/contrib/*
- scrapy/contrib_exp/*
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 000000000..8ca10109b
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Report a problem to help us improve
+---
+
+
+
+### Description
+
+[Description of the issue]
+
+### Steps to Reproduce
+
+1. [First Step]
+2. [Second Step]
+3. [and so on...]
+
+**Expected behavior:** [What you expect to happen]
+
+**Actual behavior:** [What actually happens]
+
+**Reproduces how often:** [What percentage of the time does it reproduce?]
+
+### Versions
+
+Please paste here the output of executing `scrapy version --verbose` in the command line.
+
+### Additional context
+
+Any additional information, configuration, data or output from commands that might be necessary to reproduce or understand the issue. Please try not to include screenshots of code or the command line, paste the contents as text instead. You can use [GitHub Flavored Markdown](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) to make the text look better.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..e05273fe2
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,33 @@
+---
+name: Feature request
+about: Suggest an idea for an enhancement or new feature
+---
+
+
+
+## Summary
+
+One paragraph explanation of the feature.
+
+## Motivation
+
+Why are we doing this? What use cases does it support? What is the expected outcome?
+
+## Describe alternatives you've considered
+
+A clear and concise description of the alternative solutions you've considered. Be sure to explain why Scrapy's existing customizability isn't suitable for this feature.
+
+## Additional context
+
+Any additional information about the feature request here.
diff --git a/.gitignore b/.gitignore
index 406146e5f..ff6e2ea65 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ dist
.idea
htmlcov/
.coverage
+.pytest_cache/
.coverage.*
.cache/
diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 000000000..563add75f
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,9 @@
+version: 2
+sphinx:
+ configuration: docs/conf.py
+python:
+ # For available versions, see:
+ # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
+ version: 3.7 # Keep in sync with .travis.yml
+ install:
+ - requirements: docs/requirements.txt
diff --git a/.travis.yml b/.travis.yml
index 6635f5d3b..66e1a9617 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
language: python
-sudo: false
+dist: xenial
branches:
only:
- master
@@ -7,31 +7,31 @@ branches:
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
matrix:
include:
- - python: 2.7
- env: TOXENV=py27
- - python: 2.7
- env: TOXENV=jessie
- - python: 2.7
- env: TOXENV=pypy
- - python: 2.7
- env: TOXENV=pypy3
- - python: 3.4
- env: TOXENV=py34
- - python: 3.5
- env: TOXENV=py35
- - python: 3.6
- env: TOXENV=py36
- - python: 3.6
- env: TOXENV=docs
+ - env: TOXENV=security
+ python: 3.8
+ - env: TOXENV=flake8
+ python: 3.8
+ - env: TOXENV=pypy3
+ - env: TOXENV=py35
+ python: 3.5
+ - env: TOXENV=pinned
+ python: 3.5
+ - env: TOXENV=py35-asyncio
+ python: 3.5.2
+ - env: TOXENV=py36
+ python: 3.6
+ - env: TOXENV=py37
+ python: 3.7
+ - env: TOXENV=py38
+ python: 3.8
+ - env: TOXENV=extra-deps
+ python: 3.8
+ - env: TOXENV=py38-asyncio
+ python: 3.8
+ - env: TOXENV=docs
+ python: 3.7 # Keep in sync with .readthedocs.yml
install:
- |
- if [ "$TOXENV" = "pypy" ]; then
- export PYPY_VERSION="pypy-5.9-linux_x86_64-portable"
- wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
- tar -jxf ${PYPY_VERSION}.tar.bz2
- virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
- source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
- fi
if [ "$TOXENV" = "pypy3" ]; then
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
@@ -62,4 +62,4 @@ deploy:
on:
tags: true
repo: scrapy/scrapy
- condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
+ condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 162602248..d1cd3e517 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -3,7 +3,7 @@
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
+contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
@@ -68,7 +68,7 @@ members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
+available at [http://contributor-covenant.org/version/1/4][version].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0a11b05d2..a05d07aee 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,5 +1,5 @@
The guidelines for contributing are available here:
-https://doc.scrapy.org/en/master/contributing.html
+https://docs.scrapy.org/en/master/contributing.html
Please do not abuse the issue tracker for support questions.
If your issue topic can be rephrased to "How to ...?", please use the
diff --git a/INSTALL b/INSTALL
index a3c7899c6..06e812936 100644
--- a/INSTALL
+++ b/INSTALL
@@ -1,4 +1,4 @@
For information about installing Scrapy see:
* docs/intro/install.rst (local file)
-* https://doc.scrapy.org/en/latest/intro/install.html (online version)
+* https://docs.scrapy.org/en/latest/intro/install.html (online version)
diff --git a/LICENSE b/LICENSE
index 6ead05ece..4d0a0863a 100644
--- a/LICENSE
+++ b/LICENSE
@@ -5,10 +5,10 @@ Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
- this list of conditions and the following disclaimer.
+ this list of conditions, and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
+ notice, this list of conditions, and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of Scrapy nor the names of its contributors may be used
diff --git a/MANIFEST.in b/MANIFEST.in
index 94de4f3bf..ae7db51fa 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,13 +3,24 @@ include AUTHORS
include INSTALL
include LICENSE
include MANIFEST.in
+include NEWS
+
include scrapy/VERSION
include scrapy/mime.types
+
+include codecov.yml
+include conftest.py
+include pytest.ini
+include requirements-*.txt
+include tox.ini
+
recursive-include scrapy/templates *
recursive-include scrapy license.txt
recursive-include docs *
prune docs/build
+
recursive-include extras *
recursive-include bin *
recursive-include tests *
+
global-exclude __pycache__ *.py[cod]
diff --git a/README.rst b/README.rst
index 1361eac26..7fefaeec9 100644
--- a/README.rst
+++ b/README.rst
@@ -34,13 +34,13 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
-For more information including a list of features check the Scrapy homepage at:
-https://scrapy.org
+Check the Scrapy homepage at https://scrapy.org for more information,
+including a list of features.
Requirements
============
-* Python 2.7 or Python 3.4+
+* Python 3.5+
* Works on Linux, Windows, Mac OSX, BSD
Install
@@ -50,29 +50,29 @@ The quick way::
pip install scrapy
-For more details see the install section in the documentation:
-https://doc.scrapy.org/en/latest/intro/install.html
+See the install section in the documentation at
+https://docs.scrapy.org/en/latest/intro/install.html for more details.
Documentation
=============
-Documentation is available online at https://doc.scrapy.org/ and in the ``docs``
+Documentation is available online at https://docs.scrapy.org/ and in the ``docs``
directory.
Releases
========
-You can find release notes at https://doc.scrapy.org/en/latest/news.html
+You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
Community (blog, twitter, mail list, IRC)
=========================================
-See https://scrapy.org/community/
+See https://scrapy.org/community/ for details.
Contributing
============
-See https://doc.scrapy.org/en/master/contributing.html
+See https://docs.scrapy.org/en/master/contributing.html for details.
Code of Conduct
---------------
@@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
Companies using Scrapy
======================
-See https://scrapy.org/companies/
+See https://scrapy.org/companies/ for a list.
Commercial Support
==================
-See https://scrapy.org/support/
+See https://scrapy.org/support/ for details.
diff --git a/appveyor.yml b/appveyor.yml
new file mode 100644
index 000000000..7fd636864
--- /dev/null
+++ b/appveyor.yml
@@ -0,0 +1,25 @@
+platform: x86
+version: '{branch}-{build}'
+environment:
+ matrix:
+ - PYTHON: "C:\\Python36"
+ TOX_ENV: py36
+
+branches:
+ only:
+ - master
+ - /d+\.\d+\.\d+[\w\-]*$/
+
+install:
+ - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+ - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
+ - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
+ - "pip install -U tox"
+
+build: false
+skip_tags: true
+test_script:
+ - "tox -e %TOX_ENV%"
+
+cache:
+ - '%LOCALAPPDATA%\pip\cache'
diff --git a/conftest.py b/conftest.py
index 8b4faf8fc..c0de09909 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,44 +1,51 @@
-import glob
-import six
+from pathlib import Path
+
import pytest
-from twisted import version as twisted_version
def _py_files(folder):
- return glob.glob(folder + "/*.py") + glob.glob(folder + "/*/*.py")
+ return (str(p) for p in Path(folder).rglob('*.py'))
collect_ignore = [
- # deprecated or moved modules
- "scrapy/conf.py",
- "scrapy/stats.py",
- "scrapy/project.py",
- "scrapy/utils/decorator.py",
- "scrapy/statscol.py",
- "scrapy/squeue.py",
- "scrapy/log.py",
- "scrapy/dupefilter.py",
- "scrapy/command.py",
- "scrapy/linkextractor.py",
- "scrapy/spider.py",
-
# not a test, but looks like a test
"scrapy/utils/testsite.py",
+ # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+ *_py_files("tests/CrawlerProcess")
+]
-] + _py_files("scrapy/contrib") + _py_files("scrapy/contrib_exp")
-
-if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
- collect_ignore += _py_files("scrapy/xlib/tx")
-
-
-if six.PY3:
- for line in open('tests/py3-ignores.txt'):
- file_path = line.strip()
- if file_path and file_path[0] != '#':
- collect_ignore.append(file_path)
+for line in open('tests/ignores.txt'):
+ file_path = line.strip()
+ if file_path and file_path[0] != '#':
+ collect_ignore.append(file_path)
@pytest.fixture()
def chdir(tmpdir):
"""Change to pytest-provided temporary directory"""
tmpdir.chdir()
+
+
+def pytest_collection_modifyitems(session, config, items):
+ # Avoid executing tests when executing `--flake8` flag (pytest-flake8)
+ try:
+ from pytest_flake8 import Flake8Item
+ if config.getoption('--flake8'):
+ items[:] = [item for item in items if isinstance(item, Flake8Item)]
+ except ImportError:
+ pass
+
+
+@pytest.fixture(scope='class')
+def reactor_pytest(request):
+ if not request.cls:
+ # doctests
+ return
+ request.cls.reactor_pytest = request.config.getoption("--reactor")
+ return request.cls.reactor_pytest
+
+
+@pytest.fixture(autouse=True)
+def only_asyncio(request, reactor_pytest):
+ if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
+ pytest.skip('This test is only run with --reactor=asyncio')
diff --git a/debian/scrapy.lintian-overrides b/debian/scrapy.lintian-overrides
index 955e7def0..b5de7f67d 100644
--- a/debian/scrapy.lintian-overrides
+++ b/debian/scrapy.lintian-overrides
@@ -1,2 +1 @@
new-package-should-close-itp-bug
-extra-license-file usr/share/pyshared/scrapy/xlib/pydispatch/license.txt
diff --git a/docs/Makefile b/docs/Makefile
index 187f03c4c..ff68bf1ae 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -82,6 +82,9 @@ pydoc-topics: build
@echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \
"into the Lib/ directory"
+coverage: BUILDER = coverage
+coverage: build
+
htmlview: html
$(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
os.path.realpath('build/html/index.html'))"
diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html
new file mode 100644
index 000000000..71aff8847
--- /dev/null
+++ b/docs/_tests/quotes.html
@@ -0,0 +1,281 @@
+
+
+
+ Quotes to Scrape
+
+
+
+
+
+
+
+
+
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is our choices, Harry, that show what we truly are, far more than our abilities.”
+
by J.K. Rowling
+ (about)
+
+
+
+
+
+
“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”
+
by Jane Austen
+ (about)
+
+
+
+
+
+
“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”
+
by Marilyn Monroe
+ (about)
+
+
+
+
+
+
“Try not to become a man of success. Rather become a man of value.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is better to be hated for what you are than to be loved for what you are not.”
+
by André Gide
+ (about)
+
+
+
+
+
+
“I have not failed. I've just found 10,000 ways that won't work.”
+
by Thomas A. Edison
+ (about)
+
+
+
+
+
+
“A woman is like a tea bag; you never know how strong it is until it's in hot water.”
+
by Eleanor Roosevelt
+ (about)
+
+
+
+
+
+
“A day without sunshine is like, you know, night.”
+
by Steve Martin
+ (about)
+
+
+
+
+
+
+
+
+
+
+
+
Quotes to Scrape
+
+
+
+
+
+
+
+
+
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is our choices, Harry, that show what we truly are, far more than our abilities.”
+
by J.K. Rowling
+ (about)
+
+
+
+
+
+
“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”
+
by Jane Austen
+ (about)
+
+
+
+
+
+
“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”
+
by Marilyn Monroe
+ (about)
+
+
+
+
+
+
“Try not to become a man of success. Rather become a man of value.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is better to be hated for what you are than to be loved for what you are not.”
+
by André Gide
+ (about)
+
+
+
+
+
+
“I have not failed. I've just found 10,000 ways that won't work.”
+
by Thomas A. Edison
+ (about)
+
+
+
+
+
+
“A woman is like a tea bag; you never know how strong it is until it's in hot water.”
+
by Eleanor Roosevelt
+ (about)
+
+
+
+
+
+
“A day without sunshine is like, you know, night.”
+
by Steve Martin
+ (about)
+
+
+
+
+
+
+
+
+
+
+
+
` first to see if your issue is addressed in a
well-known question
-* if you have a general question about scrapy usage, please ask it at
+* if you have a general question about Scrapy usage, please ask it at
`Stack Overflow `__
(use "scrapy" tag).
-* check the `open issues`_ to see if it has already been reported. If it has,
- don't dismiss the report, but check the ticket history and comments. If you
- have additional useful information, please leave a comment, or consider
+* check the `open issues`_ to see if the issue has already been reported. If it
+ has, don't dismiss the report, but check the ticket history and comments. If
+ you have additional useful information, please leave a comment, or consider
:ref:`sending a pull request ` with a fix.
* search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
been discussed there, or if you're not sure if what you're seeing is a bug.
- You can also ask in the `#scrapy` IRC channel.
+ You can also ask in the ``#scrapy`` IRC channel.
* write **complete, reproducible, specific bug reports**. The smaller the test
case, the better. Remember that other developers won't have your project to
@@ -99,6 +99,15 @@ Well-written patches should:
the documentation changes in the same patch. See `Documentation policies`_
below.
+* if you're adding a private API, please add a regular expression to the
+ ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new
+ private API from documentation coverage checks.
+
+ To see if your private API is skipped properly, generate a documentation
+ coverage report as follows::
+
+ tox -e docs-coverage
+
.. _submitting-patches:
Submitting patches
@@ -122,7 +131,7 @@ conversation in the `Scrapy subreddit`_ to discuss your idea first.
Sometimes there is an existing pull request for the problem you'd like to
solve, which is stalled for some reason. Often the pull request is in a
right direction, but changes are requested by Scrapy maintainers, and the
-original pull request author haven't had time to address them.
+original pull request author hasn't had time to address them.
In this case consider picking up this pull request: open
a new pull request with all commits from the original pull request, as well as
additional changes to address the raised issues. Doing so helps a lot; it is
@@ -143,7 +152,7 @@ instead of "Fix for #411". Complete titles make it easy to skim through
the issue tracker.
Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
-removal, etc) in separate commits than functional changes. This will make pull
+removal, etc) in separate commits from functional changes. This will make pull
requests easier to review and more likely to get merged.
Coding style
@@ -162,57 +171,77 @@ Scrapy:
See https://help.github.com/articles/setting-your-username-in-git/ for
setup instructions.
+.. _documentation-policies:
+
Documentation policies
======================
-* **Don't** use docstrings for documenting classes, or methods which are
- already documented in the official (sphinx) documentation. Alternatively,
- **do** provide a docstring, but make sure sphinx documentation uses
- autodoc_ extension to pull the docstring. For example, the
- :meth:`ItemLoader.add_value` method should be either
- documented only in the sphinx documentation (not it a docstring), or
- it should have a docstring which is pulled to sphinx documentation using
- autodoc_ extension.
+For reference documentation of API members (classes, methods, etc.) use
+docstrings and make sure that the Sphinx documentation uses the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
+documentation should follow docstring conventions (`PEP 257`_) and be
+IDE-friendly: short, to the point, and it may provide short examples.
-* **Do** use docstrings for documenting functions not present in the official
- (sphinx) documentation, such as functions from ``scrapy.utils`` package and
- its sub-modules.
+Other types of documentation, such as tutorials or topics, should be covered in
+files within the ``docs/`` directory. This includes documentation that is
+specific to an API member, but goes beyond API reference documentation.
-.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
+In any case, if something is covered in a docstring, use the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
+documentation instead of duplicating the docstring in files within the
+``docs/`` directory.
Tests
=====
-Tests are implemented using the `Twisted unit-testing framework`_, running
-tests requires `tox`_.
+Tests are implemented using the :doc:`Twisted unit-testing framework
+`. Running tests requires
+:doc:`tox `.
+
+.. _running-tests:
Running tests
-------------
-Make sure you have a recent enough `tox`_ installation:
+To run all tests::
- ``tox --version``
-
-If your version is older than 1.7.0, please update it first:
-
- ``pip install -U tox``
-
-To run all tests go to the root directory of Scrapy source code and run:
-
- ``tox``
+ tox
To run a specific test (say ``tests/test_loader.py``) use:
``tox -- tests/test_loader.py``
-To see coverage report install `coverage`_ (``pip install coverage``) and run:
+To run the tests on a specific :doc:`tox ` environment, use
+``-e `` with an environment name from ``tox.ini``. For example, to run
+the tests with Python 3.6 use::
+
+ tox -e py36
+
+You can also specify a comma-separated list of environments, and use :ref:`tox’s
+parallel mode ` to run the tests on multiple environments in
+parallel::
+
+ tox -e py36,py38 -p auto
+
+To pass command-line options to :doc:`pytest `, add them after
+``--`` in your call to :doc:`tox `. Using ``--`` overrides the
+default positional arguments defined in ``tox.ini``, so you must include those
+default positional arguments (``scrapy tests``) after ``--`` as well::
+
+ tox -- scrapy tests -x # stop after first failure
+
+You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
+the Python 3.6 :doc:`tox ` environment using all your CPU cores::
+
+ tox -e py36 -- scrapy tests -n auto
+
+To see coverage report install :doc:`coverage `
+(``pip install coverage``) and run:
``coverage report``
see output of ``coverage --help`` for more options like html or xml report.
-.. _coverage: https://pypi.python.org/pypi/coverage
-
Writing tests
-------------
@@ -233,9 +262,9 @@ And their unit-tests are in::
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://reddit.com/r/scrapy
-.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
.. _open issues: https://github.com/scrapy/scrapy/issues
-.. _pull request: https://help.github.com/send-pull-requests/
-.. _tox: https://pypi.python.org/pypi/tox
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
+.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
diff --git a/docs/faq.rst b/docs/faq.rst
index 7a0628f88..b65908012 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,11 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
What Python versions does Scrapy support?
-----------------------------------------
-Scrapy is supported under Python 2.7 and Python 3.4+
+Scrapy is supported under Python 3.5+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
-Python 2.6 support was dropped starting at Scrapy 0.20.
Python 3 support was added in Scrapy 1.1.
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
+Python 2 support was dropped in Scrapy 2.0.
.. note::
For Python 3 support on Windows, it is recommended to use
@@ -129,13 +129,23 @@ Does Scrapy crawl in breadth-first or depth-first order?
By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which
basically means that it crawls in `DFO order`_. This order is more convenient
-in most cases. If you do want to crawl in true `BFO order`_, you can do it by
+in most cases.
+
+If you do want to crawl in true `BFO order`_, you can do it by
setting the following settings::
DEPTH_PRIORITY = 1
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
+While pending requests are below the configured values of
+:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
+concurrently. As a result, the first few requests of a crawl rarely follow the
+desired order. Lowering those settings to ``1`` enforces the desired order, but
+it significantly slows down the crawl as a whole.
+
+
My Scrapy crawler has memory leaks. What can I do?
--------------------------------------------------
@@ -319,6 +329,38 @@ I'm scraping a XML document and my XPath selector doesn't return any items
You may need to remove namespaces. See :ref:`removing-namespaces`.
+.. _faq-split-item:
+
+How to split an item into multiple items in an item pipeline?
+-------------------------------------------------------------
+
+:ref:`Item pipelines ` cannot yield multiple items per
+input item. :ref:`Create a spider middleware `
+instead, and use its
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+method for this purpose. For example::
+
+ from copy import deepcopy
+
+ from scrapy.item import BaseItem
+
+
+ class MultiplyItemsMiddleware:
+
+ def process_spider_output(self, response, result, spider):
+ for item in result:
+ if isinstance(item, (BaseItem, dict)):
+ for _ in range(item['multiply_by']):
+ yield deepcopy(item)
+
+Does Scrapy support IPv6 addresses?
+-----------------------------------
+
+Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Note that by doing so, you lose the ability to set a specific timeout for DNS requests
+(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
+
+
.. _user agents: https://en.wikipedia.org/wiki/User_agent
.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search
diff --git a/docs/index.rst b/docs/index.rst
index 7e8c979c4..a4343b7e0 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -4,7 +4,13 @@
Scrapy |version| documentation
==============================
-This documentation contains everything you need to know about Scrapy.
+Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used
+to crawl websites and extract structured data from their pages. It can be used
+for a wide range of purposes, from data mining to monitoring and automated
+testing.
+
+.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
+.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
Getting help
============
@@ -151,8 +157,8 @@ Solving specific problems
topics/contracts
topics/practices
topics/broad-crawls
- topics/firefox
- topics/firebug
+ topics/developer-tools
+ topics/dynamic-content
topics/leaks
topics/media-pipeline
topics/deploy
@@ -164,7 +170,7 @@ Solving specific problems
Get answers to most frequently asked questions.
:doc:`topics/debug`
- Learn how to debug common problems of your scrapy spider.
+ Learn how to debug common problems of your Scrapy spider.
:doc:`topics/contracts`
Learn how to use contracts for testing your spiders.
@@ -175,11 +181,11 @@ Solving specific problems
:doc:`topics/broad-crawls`
Tune Scrapy for crawling a lot domains in parallel.
-:doc:`topics/firefox`
- Learn how to scrape with Firefox and some useful add-ons.
+:doc:`topics/developer-tools`
+ Learn how to scrape with your browser's developer tools.
-:doc:`topics/firebug`
- Learn how to scrape efficiently using Firebug.
+:doc:`topics/dynamic-content`
+ Read webpage data that is loaded dynamically.
:doc:`topics/leaks`
Learn how to find and get rid of memory leaks in your crawler.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 4a9aa3cfb..178be723c 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
Installing Scrapy
=================
-Scrapy runs on Python 2.7 and Python 3.4 or above
+Scrapy runs on Python 3.5 or above
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
@@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `,
to avoid conflicting with your system packages.
-For more detailed and platform specifics instructions, read on.
+For more detailed and platform specifics instructions, as well as
+troubleshooting information, read on.
Things that are good to know
@@ -77,9 +78,9 @@ TL;DR: We recommend installing Scrapy inside a virtual environment
on all platforms.
Python packages can be installed either globally (a.k.a system wide),
-or in user-space. We do not recommend installing scrapy system wide.
+or in user-space. We do not recommend installing Scrapy system wide.
-Instead, we recommend that you install scrapy within a so-called
+Instead, we recommend that you install Scrapy within a so-called
"virtual environment" (`virtualenv`_).
Virtualenvs allow you to not conflict with already-installed Python
system packages (which could break some of your system tools and scripts),
@@ -96,15 +97,13 @@ Check this `user guide`_ on how to create your virtualenv.
.. note::
If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
-Once you have created a virtualenv, you can install scrapy inside it with ``pip``,
+Once you have created a virtualenv, you can install Scrapy inside it with ``pip``,
just like any other Python package.
(See :ref:`platform-specific guides `
below for non-Python dependencies that you may need to install beforehand).
-Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
-
-* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
-* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
+Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
+only supports Python 3, make sure you created a Python 3 virtualenv.
.. _virtualenv: https://virtualenv.pypa.io
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
@@ -145,19 +144,15 @@ albeit with potential issues with TLS connections.
typically too old and slow to catch up with latest Scrapy.
-To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
+To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
these dependencies::
- sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
+ sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
-- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
+- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
are required for ``lxml``
- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
-If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
-
- sudo apt-get install python3 python3-dev
-
Inside a :ref:`virtualenv `,
you can install Scrapy with ``pip`` after that::
@@ -230,26 +225,54 @@ PyPy
We recommend using the latest PyPy version. The version tested is 5.9.0.
For PyPy3, only Linux installation was tested.
-Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
+Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
This means that these dependecies will be built during installation.
On OS X, you are likely to face an issue with building Cryptography dependency,
solution to this problem is described
`here `_,
that is to ``brew install openssl`` and then export the flags that this command
-recommends (only needed when installing scrapy). Installing on Linux has no special
+recommends (only needed when installing Scrapy). Installing on Linux has no special
issues besides installing build dependencies.
-Installing scrapy with PyPy on Windows is not tested.
+Installing Scrapy with PyPy on Windows is not tested.
-You can check that scrapy is installed correctly by running ``scrapy bench``.
+You can check that Scrapy is installed correctly by running ``scrapy bench``.
If this command gives errors such as
``TypeError: ... got 2 unexpected keyword arguments``, this means
that setuptools was unable to pick up one PyPy-specific dependency.
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
+.. _intro-install-troubleshooting:
+
+Troubleshooting
+===============
+
+AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+----------------------------------------------------------------
+
+After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an
+exception with the following traceback::
+
+ […]
+ File "[…]/site-packages/twisted/protocols/tls.py", line 63, in
+ from twisted.internet._sslverify import _setAcceptableProtocols
+ File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in
+ TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1,
+ AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
+
+The reason you get this exception is that your system or virtual environment
+has a version of pyOpenSSL that your version of Twisted does not support.
+
+To install a version of pyOpenSSL that your version of Twisted supports,
+reinstall Twisted with the :code:`tls` extra option::
+
+ pip install twisted[tls]
+
+For details, see `Issue #2473 `_.
+
.. _Python: https://www.python.org/
.. _pip: https://pip.pypa.io/en/latest/installing/
-.. _lxml: http://lxml.de/
+.. _lxml: https://lxml.de/index.html
.. _parsel: https://pypi.python.org/pypi/parsel
.. _w3lib: https://pypi.python.org/pypi/w3lib
.. _twisted: https://twistedmatrix.com/
@@ -261,5 +284,5 @@ To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
.. _zsh: https://www.zsh.org/
.. _Scrapinghub: https://scrapinghub.com
.. _Anaconda: https://docs.anaconda.com/anaconda/
-.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
+.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
.. _conda-forge: https://conda-forge.org/
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 6f1c2c43f..01986b594 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -26,7 +26,7 @@ http://quotes.toscrape.com, following the pagination::
class QuotesSpider(scrapy.Spider):
- name = "quotes"
+ name = 'quotes'
start_urls = [
'http://quotes.toscrape.com/tag/humor/',
]
@@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.xpath('span/small/text()').extract_first(),
+ 'author': quote.xpath('span/small/text()').get(),
+ 'text': quote.css('span.text::text').get(),
}
- next_page = response.css('li.next a::attr("href")').extract_first()
+ next_page = response.css('li.next a::attr("href")').get()
if next_page is not None:
yield response.follow(next_page, self.parse)
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 20538e90f..ee10048b5 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -22,19 +22,27 @@ Scrapy is written in Python_. If you're new to the language you might want to
start by getting an idea of what the language is like, to get the most out of
Scrapy.
-If you're already familiar with other languages, and want to learn Python
-quickly, we recommend reading through `Dive Into Python 3`_. Alternatively,
-you can follow the `Python Tutorial`_.
+If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
-If you're new to programming and want to start with Python, you may find useful
-the online book `Learn Python The Hard Way`_. You can also take a look at `this
-list of Python resources for non-programmers`_.
+If you're new to programming and want to start with Python, the following books
+may be useful to you:
+
+* `Automate the Boring Stuff With Python`_
+
+* `How To Think Like a Computer Scientist`_
+
+* `Learn Python 3 The Hard Way`_
+
+You can also take a look at `this list of Python resources for non-programmers`_,
+as well as the `suggested resources in the learnpython-subreddit`_.
.. _Python: https://www.python.org/
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
-.. _Dive Into Python 3: http://www.diveintopython3.net
.. _Python Tutorial: https://docs.python.org/3/tutorial
-.. _Learn Python The Hard Way: https://learnpythonthehardway.org/book/
+.. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
+.. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/
+.. _Learn Python 3 The Hard Way: https://learnpythonthehardway.org/python3/
+.. _suggested resources in the learnpython-subreddit: https://www.reddit.com/r/learnpython/wiki/index#wiki_new_to_python.3F
Creating a project
@@ -70,9 +78,9 @@ Our first Spider
Spiders are classes that you define and that Scrapy uses to scrape information
from a website (or a group of websites). They must subclass
-:class:`scrapy.Spider` and define the initial requests to make, optionally how
-to follow links in the pages, and how to parse the downloaded page content to
-extract data.
+:class:`~scrapy.spiders.Spider` and define the initial requests to make,
+optionally how to follow links in the pages, and how to parse the downloaded
+page content to extract data.
This is the code for our first Spider. Save it in a file named
``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
@@ -197,7 +205,7 @@ Extracting data
---------------
The best way to learn how to extract data with Scrapy is trying selectors
-using the shell :ref:`Scrapy shell `. Run::
+using the :ref:`Scrapy shell `. Run::
scrapy shell 'http://quotes.toscrape.com/page/1/'
@@ -227,13 +235,16 @@ You will see something like::
[s] shelp() Shell help (print this help)
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
[s] view(response) View response in a browser
- >>>
Using the shell, you can try selecting elements using `CSS`_ with the response
-object::
+object:
- >>> response.css('title')
- []
+.. invisible-code-block: python
+
+ response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
+
+>>> response.css('title')
+[]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
@@ -241,54 +252,55 @@ The result of running ``response.css('title')`` is a list-like object called
and allow you to run further queries to fine-grain the selection or extract the
data.
-To extract the text from the title above, you can do::
+To extract the text from the title above, you can do:
- >>> response.css('title::text').extract()
- ['Quotes to Scrape']
+>>> response.css('title::text').getall()
+['Quotes to Scrape']
There are two things to note here: one is that we've added ``::text`` to the
CSS query, to mean we want to select only the text elements directly inside
```` element. If we don't specify ``::text``, we'd get the full title
-element, including its tags::
+element, including its tags:
- >>> response.css('title').extract()
- ['<title>Quotes to Scrape ']
+>>> response.css('title').getall()
+['Quotes to Scrape ']
-The other thing is that the result of calling ``.extract()`` is a list, because
-we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When
-you know you just want the first result, as in this case, you can do::
+The other thing is that the result of calling ``.getall()`` is a list: it is
+possible that a selector returns more than one result, so we extract them all.
+When you know you just want the first result, as in this case, you can do:
- >>> response.css('title::text').extract_first()
- 'Quotes to Scrape'
+>>> response.css('title::text').get()
+'Quotes to Scrape'
-As an alternative, you could've written::
+As an alternative, you could've written:
- >>> response.css('title::text')[0].extract()
- 'Quotes to Scrape'
+>>> response.css('title::text')[0].get()
+'Quotes to Scrape'
-However, using ``.extract_first()`` avoids an ``IndexError`` and returns
-``None`` when it doesn't find any element matching the selection.
+However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
+instance avoids an ``IndexError`` and returns ``None`` when it doesn't
+find any element matching the selection.
There's a lesson here: for most scraping code, you want it to be resilient to
errors due to things not being found on a page, so that even if some parts fail
to be scraped, you can at least get **some** data.
-Besides the :meth:`~scrapy.selector.Selector.extract` and
-:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
-the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
-expressions`::
+Besides the :meth:`~scrapy.selector.SelectorList.getall` and
+:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
+the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
+expressions`_:
- >>> response.css('title::text').re(r'Quotes.*')
- ['Quotes to Scrape']
- >>> response.css('title::text').re(r'Q\w+')
- ['Quotes']
- >>> response.css('title::text').re(r'(\w+) to (\w+)')
- ['Quotes', 'Scrape']
+>>> response.css('title::text').re(r'Quotes.*')
+['Quotes to Scrape']
+>>> response.css('title::text').re(r'Q\w+')
+['Quotes']
+>>> response.css('title::text').re(r'(\w+) to (\w+)')
+['Quotes', 'Scrape']
In order to find the proper CSS selectors to use, you might find useful opening
the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools or extensions like Firebug (see
-sections about :ref:`topics-firebug` and :ref:`topics-firefox`).
+You can use your browser's developer tools to inspect the HTML and come up
+with a selector (see :ref:`topics-developer-tools`).
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
visually selected elements, which works in many browsers.
@@ -300,12 +312,12 @@ visually selected elements, which works in many browsers.
XPath: a brief intro
^^^^^^^^^^^^^^^^^^^^
-Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
+Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
- >>> response.xpath('//title')
- []
- >>> response.xpath('//title/text()').extract_first()
- 'Quotes to Scrape'
+>>> response.xpath('//title')
+[]
+>>> response.xpath('//title/text()').get()
+'Quotes to Scrape'
XPath expressions are very powerful, and are the foundation of Scrapy
Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@@ -360,45 +372,53 @@ we want::
$ scrapy shell 'http://quotes.toscrape.com'
-We get a list of selectors for the quote HTML elements with::
+We get a list of selectors for the quote HTML elements with:
- >>> response.css("div.quote")
+>>> response.css("div.quote")
+[,
+ ,
+ ...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
-variable, so that we can run our CSS selectors directly on a particular quote::
+variable, so that we can run our CSS selectors directly on a particular quote:
- >>> quote = response.css("div.quote")[0]
+>>> quote = response.css("div.quote")[0]
-Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
-using the ``quote`` object we just created::
+Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
+using the ``quote`` object we just created:
- >>> title = quote.css("span.text::text").extract_first()
- >>> title
- '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
- >>> author = quote.css("small.author::text").extract_first()
- >>> author
- 'Albert Einstein'
+>>> text = quote.css("span.text::text").get()
+>>> text
+'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
+>>> author = quote.css("small.author::text").get()
+>>> author
+'Albert Einstein'
-Given that the tags are a list of strings, we can use the ``.extract()`` method
-to get all of them::
+Given that the tags are a list of strings, we can use the ``.getall()`` method
+to get all of them:
- >>> tags = quote.css("div.tags a.tag::text").extract()
- >>> tags
- ['change', 'deep-thoughts', 'thinking', 'world']
+>>> tags = quote.css("div.tags a.tag::text").getall()
+>>> tags
+['change', 'deep-thoughts', 'thinking', 'world']
+
+.. invisible-code-block: python
+
+ from sys import version_info
+
+.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
Having figured out how to extract each bit, we can now iterate over all the
-quotes elements and put them together into a Python dictionary::
+quotes elements and put them together into a Python dictionary:
- >>> for quote in response.css("div.quote"):
- ... text = quote.css("span.text::text").extract_first()
- ... author = quote.css("small.author::text").extract_first()
- ... tags = quote.css("div.tags a.tag::text").extract()
- ... print(dict(text=text, author=author, tags=tags))
- {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
- {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
- ... a few more of these, omitted for brevity
- >>>
+>>> for quote in response.css("div.quote"):
+... text = quote.css("span.text::text").get()
+... author = quote.css("small.author::text").get()
+... tags = quote.css("div.tags a.tag::text").getall()
+... print(dict(text=text, author=author, tags=tags))
+{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
+{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
+...
Extracting data in our spider
-----------------------------
@@ -424,9 +444,9 @@ in the callback, as you can see below::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('small.author::text').extract_first(),
- 'tags': quote.css('div.tags a.tag::text').extract(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('small.author::text').get(),
+ 'tags': quote.css('div.tags a.tag::text').getall(),
}
If you run this spider, it will output the extracted data with the log::
@@ -496,17 +516,23 @@ markup:
-We can try extracting it in the shell::
+We can try extracting it in the shell:
- >>> response.css('li.next a').extract_first()
- 'Next → '
+>>> response.css('li.next a').get()
+'Next → '
This gets the anchor element, but we want the attribute ``href``. For that,
-Scrapy supports a CSS extension that let's you select the attribute contents,
-like this::
+Scrapy supports a CSS extension that lets you select the attribute contents,
+like this:
- >>> response.css('li.next a::attr(href)').extract_first()
- '/page/2/'
+>>> response.css('li.next a::attr(href)').get()
+'/page/2/'
+
+There is also an ``attrib`` property available
+(see :ref:`selecting-attributes` for more):
+
+>>> response.css('li.next a').attrib['href']
+'/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
@@ -523,12 +549,12 @@ page, extracting data from it::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('small.author::text').extract_first(),
- 'tags': quote.css('div.tags a.tag::text').extract(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('small.author::text').get(),
+ 'tags': quote.css('div.tags a.tag::text').getall(),
}
- next_page = response.css('li.next a::attr(href)').extract_first()
+ next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
next_page = response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
@@ -574,12 +600,12 @@ As a shortcut for creating Request objects you can use
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('span small::text').extract_first(),
- 'tags': quote.css('div.tags a.tag::text').extract(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('span small::text').get(),
+ 'tags': quote.css('div.tags a.tag::text').getall(),
}
- next_page = response.css('li.next a::attr(href)').extract_first()
+ next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
yield response.follow(next_page, callback=self.parse)
@@ -590,21 +616,25 @@ instance; you still have to yield this Request.
You can also pass a selector to ``response.follow`` instead of a string;
this selector should extract necessary attributes::
- for href in response.css('li.next a::attr(href)'):
+ for href in response.css('ul.pager a::attr(href)'):
yield response.follow(href, callback=self.parse)
For ```` elements there is a shortcut: ``response.follow`` uses their href
attribute automatically. So the code can be shortened further::
- for a in response.css('li.next a'):
+ for a in response.css('ul.pager a'):
yield response.follow(a, callback=self.parse)
-.. note::
+To create multiple requests from an iterable, you can use
+:meth:`response.follow_all ` instead::
+
+ anchors = response.css('ul.pager a')
+ yield from response.follow_all(anchors, callback=self.parse)
+
+or, shortening it further::
+
+ yield from response.follow_all(css='ul.pager a', callback=self.parse)
- ``response.follow(response.css('li.next a'))`` is not valid because
- ``response.css`` returns a list-like object with selectors for all results,
- not a single selector. A ``for`` loop like in the example above, or
- ``response.follow(response.css('li.next a')[0])`` is fine.
More examples and patterns
--------------------------
@@ -621,17 +651,15 @@ this time for scraping author information::
start_urls = ['http://quotes.toscrape.com/']
def parse(self, response):
- # follow links to author pages
- for href in response.css('.author + a::attr(href)'):
- yield response.follow(href, self.parse_author)
+ author_page_links = response.css('.author + a')
+ yield from response.follow_all(author_page_links, self.parse_author)
- # follow pagination links
- for href in response.css('li.next a::attr(href)'):
- yield response.follow(href, self.parse)
+ pagination_links = response.css('li.next a')
+ yield from response.follow_all(pagination_links, self.parse)
def parse_author(self, response):
def extract_with_css(query):
- return response.css(query).extract_first().strip()
+ return response.css(query).get(default='').strip()
yield {
'name': extract_with_css('h3.author-title::text'),
@@ -643,8 +671,10 @@ This spider will start from the main page, it will follow all the links to the
authors pages calling the ``parse_author`` callback for each of them, and also
the pagination links with the ``parse`` callback as we saw before.
-Here we're passing callbacks to ``response.follow`` as positional arguments
-to make the code shorter; it also works for ``scrapy.Request``.
+Here we're passing callbacks to
+:meth:`response.follow_all ` as positional
+arguments to make the code shorter; it also works for
+:class:`~scrapy.http.Request`.
The ``parse_author`` callback defines a helper function to extract and cleanup the
data from a CSS query and yields the Python dict with the author data.
@@ -700,11 +730,11 @@ with a specific tag, building the URL based on the argument::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').extract_first(),
- 'author': quote.css('small.author::text').extract_first(),
+ 'text': quote.css('span.text::text').get(),
+ 'author': quote.css('small.author::text').get(),
}
- next_page = response.css('li.next a::attr(href)').extract_first()
+ next_page = response.css('li.next a::attr(href)').get()
if next_page is not None:
yield response.follow(next_page, self.parse)
@@ -728,4 +758,3 @@ modeling the scraped data. If you prefer to play with an example project, check
the :ref:`intro-examples` section.
.. _JSON: https://en.wikipedia.org/wiki/JSON
-.. _dirbot: https://github.com/scrapy/dirbot
diff --git a/docs/news.rst b/docs/news.rst
index 1629510b2..6d0d4b4ee 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,805 @@
Release notes
=============
+.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
+ planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
+
+.. _release-1.8.0:
+
+Scrapy 1.8.0 (2019-10-28)
+-------------------------
+
+Highlights:
+
+* Dropped Python 3.4 support and updated minimum requirements; made Python 3.8
+ support official
+* New :meth:`Request.from_curl ` class method
+* New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings
+* New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
+ :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Python 3.4 is no longer supported, and some of the minimum requirements of
+ Scrapy have also changed:
+
+ * cssselect_ 0.9.1
+ * cryptography_ 2.0
+ * lxml_ 3.5.0
+ * pyOpenSSL_ 16.2.0
+ * queuelib_ 1.4.2
+ * service_identity_ 16.0.0
+ * six_ 1.10.0
+ * Twisted_ 17.9.0 (16.0.0 with Python 2)
+ * zope.interface_ 4.1.3
+
+ (:issue:`3892`)
+
+* ``JSONRequest`` is now called :class:`~scrapy.http.JsonRequest` for
+ consistency with similar classes (:issue:`3929`, :issue:`3982`)
+
+* If you are using a custom context factory
+ (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must
+ accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers``
+ (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
+
+* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
+ into lists:
+
+ >>> item = MyItem()
+ >>> item['field'] = 'value1'
+ >>> loader = ItemLoader(item=item)
+ >>> item['field']
+ ['value1']
+
+ This is needed to allow adding values to existing fields
+ (``loader.add_value('field', 'value2')``).
+
+ (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`,
+ :issue:`4036`)
+
+See also :ref:`1.8-deprecation-removals` below.
+
+
+New features
+~~~~~~~~~~~~
+
+* A new :meth:`Request.from_curl ` class
+ method allows :ref:`creating a request from a cURL command
+ ` (:issue:`2985`, :issue:`3862`)
+
+* A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_
+ parser to use. It includes built-in support for
+ :ref:`RobotFileParser `,
+ :ref:`Protego ` (default), :ref:`Reppy `, and
+ :ref:`Robotexclusionrulesparser `, and allows you to
+ :ref:`implement support for additional parsers
+ ` (:issue:`754`, :issue:`2669`,
+ :issue:`3796`, :issue:`3935`, :issue:`3969`, :issue:`4006`)
+
+* A new :setting:`ROBOTSTXT_USER_AGENT` setting allows defining a separate
+ user agent string to use for robots.txt_ parsing (:issue:`3931`,
+ :issue:`3966`)
+
+* :class:`~scrapy.spiders.Rule` no longer requires a :class:`LinkExtractor
+ ` parameter
+ (:issue:`781`, :issue:`4016`)
+
+* Use the new :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting to customize
+ the TLS/SSL ciphers used by the default HTTP/1.1 downloader (:issue:`3392`,
+ :issue:`3442`)
+
+* Set the new :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` setting to
+ ``True`` to enable debug-level messages about TLS connection parameters
+ after establishing HTTPS connections (:issue:`2111`, :issue:`3450`)
+
+* Callbacks that receive keyword arguments
+ (see :attr:`Request.cb_kwargs `) can now be
+ tested using the new :class:`@cb_kwargs
+ `
+ :ref:`spider contract ` (:issue:`3985`, :issue:`3988`)
+
+* When a :class:`@scrapes ` spider
+ contract fails, all missing fields are now reported (:issue:`766`,
+ :issue:`3939`)
+
+* :ref:`Custom log formats ` can now drop messages by
+ having the corresponding methods of the configured :setting:`LOG_FORMATTER`
+ return ``None`` (:issue:`3984`, :issue:`3987`)
+
+* A much improved completion definition is now available for Zsh_
+ (:issue:`4069`)
+
+
+Bug fixes
+~~~~~~~~~
+
+* :meth:`ItemLoader.load_item() ` no
+ longer makes later calls to :meth:`ItemLoader.get_output_value()
+ ` or
+ :meth:`ItemLoader.load_item() ` return
+ empty data (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`,
+ :issue:`3998`, :issue:`4036`)
+
+* Fixed :class:`~scrapy.statscollectors.DummyStatsCollector` raising a
+ :exc:`TypeError` exception (:issue:`4007`, :issue:`4052`)
+
+* :meth:`FilesPipeline.file_path
+ ` and
+ :meth:`ImagesPipeline.file_path
+ ` no longer choose
+ file extensions that are not `registered with IANA`_ (:issue:`1287`,
+ :issue:`3953`, :issue:`3954`)
+
+* When using botocore_ to persist files in S3, all botocore-supported headers
+ are properly mapped now (:issue:`3904`, :issue:`3905`)
+
+* FTP passwords in :setting:`FEED_URI` containing percent-escaped characters
+ are now properly decoded (:issue:`3941`)
+
+* A memory-handling and error-handling issue in
+ :func:`scrapy.utils.ssl.get_temp_key_info` has been fixed (:issue:`3920`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+* The documentation now covers how to define and configure a :ref:`custom log
+ format ` (:issue:`3616`, :issue:`3660`)
+
+* API documentation added for :class:`~scrapy.exporters.MarshalItemExporter`
+ and :class:`~scrapy.exporters.PythonItemExporter` (:issue:`3973`)
+
+* API documentation added for :class:`~scrapy.item.BaseItem` and
+ :class:`~scrapy.item.ItemMeta` (:issue:`3999`)
+
+* Minor documentation fixes (:issue:`2998`, :issue:`3398`, :issue:`3597`,
+ :issue:`3894`, :issue:`3934`, :issue:`3978`, :issue:`3993`, :issue:`4022`,
+ :issue:`4028`, :issue:`4033`, :issue:`4046`, :issue:`4050`, :issue:`4055`,
+ :issue:`4056`, :issue:`4061`, :issue:`4072`, :issue:`4071`, :issue:`4079`,
+ :issue:`4081`, :issue:`4089`, :issue:`4093`)
+
+
+.. _1.8-deprecation-removals:
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+* ``scrapy.xlib`` has been removed (:issue:`4015`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+* The LevelDB_ storage backend
+ (``scrapy.extensions.httpcache.LeveldbCacheStorage``) of
+ :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is
+ deprecated (:issue:`4085`, :issue:`4092`)
+
+* Use of the undocumented ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
+ variable is deprecated (:issue:`3910`)
+
+* ``scrapy.item.DictItem`` is deprecated, use :class:`~scrapy.item.Item`
+ instead (:issue:`3999`)
+
+
+Other changes
+~~~~~~~~~~~~~
+
+* Minimum versions of optional Scrapy requirements that are covered by
+ continuous integration tests have been updated:
+
+ * botocore_ 1.3.23
+ * Pillow_ 3.4.2
+
+ Lower versions of these optional requirements may work, but it is not
+ guaranteed (:issue:`3892`)
+
+* GitHub templates for bug reports and feature requests (:issue:`3126`,
+ :issue:`3471`, :issue:`3749`, :issue:`3754`)
+
+* Continuous integration fixes (:issue:`3923`)
+
+* Code cleanup (:issue:`3391`, :issue:`3907`, :issue:`3946`, :issue:`3950`,
+ :issue:`4023`, :issue:`4031`)
+
+
+.. _release-1.7.4:
+
+Scrapy 1.7.4 (2019-10-21)
+-------------------------
+
+Revert the fix for :issue:`3804` (:issue:`3819`), which has a few undesired
+side effects (:issue:`3897`, :issue:`3976`).
+
+As a result, when an item loader is initialized with an item,
+:meth:`ItemLoader.load_item() ` once again
+makes later calls to :meth:`ItemLoader.get_output_value()
+` or :meth:`ItemLoader.load_item()
+` return empty data.
+
+
+.. _release-1.7.3:
+
+Scrapy 1.7.3 (2019-08-01)
+-------------------------
+
+Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
+
+
+.. _release-1.7.2:
+
+Scrapy 1.7.2 (2019-07-23)
+-------------------------
+
+Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
+
+
+.. _release-1.7.1:
+
+Scrapy 1.7.1 (2019-07-18)
+-------------------------
+
+Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
+
+
+.. _release-1.7.0:
+
+Scrapy 1.7.0 (2019-07-18)
+-------------------------
+
+.. note:: Make sure you install Scrapy 1.7.1. The Scrapy 1.7.0 package in PyPI
+ is the result of an erroneous commit tagging and does not include all
+ the changes described below.
+
+Highlights:
+
+* Improvements for crawls targeting multiple domains
+* A cleaner way to pass arguments to callbacks
+* A new class for JSON requests
+* Improvements for rule-based spiders
+* New features for feed exports
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``429`` is now part of the :setting:`RETRY_HTTP_CODES` setting by default
+
+ This change is **backward incompatible**. If you don’t want to retry
+ ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
+
+* :class:`~scrapy.crawler.Crawler`,
+ :class:`CrawlerRunner.crawl ` and
+ :class:`CrawlerRunner.create_crawler `
+ no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
+ only accept a :class:`~scrapy.spiders.Spider` subclass now.
+
+ :class:`~scrapy.spiders.Spider` subclass instances were never meant to
+ work, and they were not working as one would expect: instead of using the
+ passed :class:`~scrapy.spiders.Spider` subclass instance, their
+ :class:`~scrapy.spiders.Spider.from_crawler` method was called to generate
+ a new instance.
+
+* Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting
+ may stop working. Scheduler priority queue classes now need to handle
+ :class:`~scrapy.http.Request` objects instead of arbitrary Python data
+ structures.
+
+See also :ref:`1.7-deprecation-removals` below.
+
+
+New features
+~~~~~~~~~~~~
+
+* A new scheduler priority queue,
+ ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
+ :ref:`enabled ` for a significant
+ scheduling improvement on crawls targetting multiple web domains, at the
+ cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
+
+* A new :attr:`Request.cb_kwargs ` attribute
+ provides a cleaner way to pass keyword arguments to callback methods
+ (:issue:`1138`, :issue:`3563`)
+
+* A new :class:`JSONRequest ` class offers a more
+ convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
+
+* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
+ ``__init__`` method now receives the :class:`~scrapy.http.Response` object that
+ originated the request as its second argument (:issue:`3682`)
+
+* A new ``restrict_text`` parameter for the
+ :attr:`LinkExtractor `
+ ``__init__`` method allows filtering links by linking text (:issue:`3622`,
+ :issue:`3635`)
+
+* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
+ for feeds exported to Amazon S3 (:issue:`3607`)
+
+* A new :setting:`FEED_STORAGE_FTP_ACTIVE` setting allows using FTP’s active
+ connection mode for feeds exported to FTP servers (:issue:`3829`)
+
+* A new :setting:`METAREFRESH_IGNORE_TAGS` setting allows overriding which
+ HTML tags are ignored when searching a response for HTML meta tags that
+ trigger a redirect (:issue:`1422`, :issue:`3768`)
+
+* A new :reqmeta:`redirect_reasons` request meta key exposes the reason
+ (status code, meta refresh) behind every followed redirect (:issue:`3581`,
+ :issue:`3687`)
+
+* The ``SCRAPY_CHECK`` variable is now set to the ``true`` string during runs
+ of the :command:`check` command, which allows :ref:`detecting contract
+ check runs from code ` (:issue:`3704`,
+ :issue:`3739`)
+
+* A new :meth:`Item.deepcopy() ` method makes it
+ easier to :ref:`deep-copy items ` (:issue:`1493`,
+ :issue:`3671`)
+
+* :class:`~scrapy.extensions.corestats.CoreStats` also logs
+ ``elapsed_time_seconds`` now (:issue:`3638`)
+
+* Exceptions from :class:`~scrapy.loader.ItemLoader` :ref:`input and output
+ processors ` are now more verbose
+ (:issue:`3836`, :issue:`3840`)
+
+* :class:`~scrapy.crawler.Crawler`,
+ :class:`CrawlerRunner.crawl ` and
+ :class:`CrawlerRunner.create_crawler `
+ now fail gracefully if they receive a :class:`~scrapy.spiders.Spider`
+ subclass instance instead of the subclass itself (:issue:`2283`,
+ :issue:`3610`, :issue:`3872`)
+
+
+Bug fixes
+~~~~~~~~~
+
+* :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
+ is now also invoked for generators (:issue:`220`, :issue:`2061`)
+
+* System exceptions like KeyboardInterrupt_ are no longer caught
+ (:issue:`3726`)
+
+* :meth:`ItemLoader.load_item() ` no
+ longer makes later calls to :meth:`ItemLoader.get_output_value()
+ ` or
+ :meth:`ItemLoader.load_item() ` return
+ empty data (:issue:`3804`, :issue:`3819`)
+
+* The images pipeline (:class:`~scrapy.pipelines.images.ImagesPipeline`) no
+ longer ignores these Amazon S3 settings: :setting:`AWS_ENDPOINT_URL`,
+ :setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`
+ (:issue:`3625`)
+
+* Fixed a memory leak in ``scrapy.pipelines.media.MediaPipeline`` affecting,
+ for example, non-200 responses and exceptions from custom middlewares
+ (:issue:`3813`)
+
+* Requests with private callbacks are now correctly unserialized from disk
+ (:issue:`3790`)
+
+* :meth:`FormRequest.from_response() `
+ now handles invalid methods like major web browsers (:issue:`3777`,
+ :issue:`3794`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+* A new topic, :ref:`topics-dynamic-content`, covers recommended approaches
+ to read dynamically-loaded data (:issue:`3703`)
+
+* :ref:`topics-broad-crawls` now features information about memory usage
+ (:issue:`1264`, :issue:`3866`)
+
+* The documentation of :class:`~scrapy.spiders.Rule` now covers how to access
+ the text of a link when using :class:`~scrapy.spiders.CrawlSpider`
+ (:issue:`3711`, :issue:`3712`)
+
+* A new section, :ref:`httpcache-storage-custom`, covers writing a custom
+ cache storage backend for
+ :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
+ (:issue:`3683`, :issue:`3692`)
+
+* A new :ref:`FAQ ` entry, :ref:`faq-split-item`, explains what to do
+ when you want to split an item into multiple items from an item pipeline
+ (:issue:`2240`, :issue:`3672`)
+
+* Updated the :ref:`FAQ entry about crawl order ` to explain why
+ the first few requests rarely follow the desired order (:issue:`1739`,
+ :issue:`3621`)
+
+* The :setting:`LOGSTATS_INTERVAL` setting (:issue:`3730`), the
+ :meth:`FilesPipeline.file_path `
+ and
+ :meth:`ImagesPipeline.file_path `
+ methods (:issue:`2253`, :issue:`3609`) and the
+ :meth:`Crawler.stop() ` method (:issue:`3842`)
+ are now documented
+
+* Some parts of the documentation that were confusing or misleading are now
+ clearer (:issue:`1347`, :issue:`1789`, :issue:`2289`, :issue:`3069`,
+ :issue:`3615`, :issue:`3626`, :issue:`3668`, :issue:`3670`, :issue:`3673`,
+ :issue:`3728`, :issue:`3762`, :issue:`3861`, :issue:`3882`)
+
+* Minor documentation fixes (:issue:`3648`, :issue:`3649`, :issue:`3662`,
+ :issue:`3674`, :issue:`3676`, :issue:`3694`, :issue:`3724`, :issue:`3764`,
+ :issue:`3767`, :issue:`3791`, :issue:`3797`, :issue:`3806`, :issue:`3812`)
+
+.. _1.7-deprecation-removals:
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+The following deprecated APIs have been removed (:issue:`3578`):
+
+* ``scrapy.conf`` (use :attr:`Crawler.settings
+ `)
+
+* From ``scrapy.core.downloader.handlers``:
+
+ * ``http.HttpDownloadHandler`` (use ``http10.HTTP10DownloadHandler``)
+
+* ``scrapy.loader.ItemLoader._get_values`` (use ``_get_xpathvalues``)
+
+* ``scrapy.loader.XPathItemLoader`` (use :class:`~scrapy.loader.ItemLoader`)
+
+* ``scrapy.log`` (see :ref:`topics-logging`)
+
+* From ``scrapy.pipelines``:
+
+ * ``files.FilesPipeline.file_key`` (use ``file_path``)
+
+ * ``images.ImagesPipeline.file_key`` (use ``file_path``)
+
+ * ``images.ImagesPipeline.image_key`` (use ``file_path``)
+
+ * ``images.ImagesPipeline.thumb_key`` (use ``thumb_path``)
+
+* From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``:
+
+ * ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+ * ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+ * ``XPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+ * ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`)
+
+* From ``scrapy.selector.csstranslator``:
+
+ * ``ScrapyGenericTranslator`` (use parsel.csstranslator.GenericTranslator_)
+
+ * ``ScrapyHTMLTranslator`` (use parsel.csstranslator.HTMLTranslator_)
+
+ * ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_)
+
+* From :class:`~scrapy.selector.Selector`:
+
+ * ``_root`` (both the ``__init__`` method argument and the object property, use
+ ``root``)
+
+ * ``extract_unquoted`` (use ``getall``)
+
+ * ``select`` (use ``xpath``)
+
+* From :class:`~scrapy.selector.SelectorList`:
+
+ * ``extract_unquoted`` (use ``getall``)
+
+ * ``select`` (use ``xpath``)
+
+ * ``x`` (use ``xpath``)
+
+* ``scrapy.spiders.BaseSpider`` (use :class:`~scrapy.spiders.Spider`)
+
+* From :class:`~scrapy.spiders.Spider` (and subclasses):
+
+ * ``DOWNLOAD_DELAY`` (use :ref:`download_delay
+ `)
+
+ * ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`)
+
+* ``scrapy.spiders.spiders`` (use :class:`~scrapy.spiderloader.SpiderLoader`)
+
+* ``scrapy.telnet`` (use :mod:`scrapy.extensions.telnet`)
+
+* From ``scrapy.utils.python``:
+
+ * ``str_to_unicode`` (use ``to_unicode``)
+
+ * ``unicode_to_str`` (use ``to_bytes``)
+
+* ``scrapy.utils.response.body_or_str``
+
+The following deprecated settings have also been removed (:issue:`3578`):
+
+* ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+* The ``queuelib.PriorityQueue`` value for the
+ :setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use
+ ``scrapy.pqueues.ScrapyPriorityQueue`` instead.
+
+* ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that
+ do not accept two arguments are deprecated.
+
+* The following modules are deprecated:
+
+ * ``scrapy.utils.http`` (use `w3lib.http`_)
+
+ * ``scrapy.utils.markup`` (use `w3lib.html`_)
+
+ * ``scrapy.utils.multipart`` (use `urllib3`_)
+
+* The ``scrapy.utils.datatypes.MergeDict`` class is deprecated for Python 3
+ code bases. Use :class:`~collections.ChainMap` instead. (:issue:`3878`)
+
+* The ``scrapy.utils.gz.is_gzipped`` function is deprecated. Use
+ ``scrapy.utils.gz.gzip_magic_number`` instead.
+
+.. _urllib3: https://urllib3.readthedocs.io/en/latest/index.html
+.. _w3lib.html: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.html
+.. _w3lib.http: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.http
+
+
+Other changes
+~~~~~~~~~~~~~
+
+* It is now possible to run all tests from the same tox_ environment in
+ parallel; the documentation now covers :ref:`this and other ways to run
+ tests ` (:issue:`3707`)
+
+* It is now possible to generate an API documentation coverage report
+ (:issue:`3806`, :issue:`3810`, :issue:`3860`)
+
+* The :ref:`documentation policies ` now require
+ docstrings_ (:issue:`3701`) that follow `PEP 257`_ (:issue:`3748`)
+
+* Internal fixes and cleanup (:issue:`3629`, :issue:`3643`, :issue:`3684`,
+ :issue:`3698`, :issue:`3734`, :issue:`3735`, :issue:`3736`, :issue:`3737`,
+ :issue:`3809`, :issue:`3821`, :issue:`3825`, :issue:`3827`, :issue:`3833`,
+ :issue:`3857`, :issue:`3877`)
+
+.. _release-1.6.0:
+
+Scrapy 1.6.0 (2019-01-30)
+-------------------------
+
+Highlights:
+
+* better Windows support;
+* Python 3.7 compatibility;
+* big documentation improvements, including a switch
+ from ``.extract_first()`` + ``.extract()`` API to ``.get()`` + ``.getall()``
+ API;
+* feed exports, FilePipeline and MediaPipeline improvements;
+* better extensibility: :signal:`item_error` and
+ :signal:`request_reached_downloader` signals; ``from_crawler`` support
+ for feed exporters, feed storages and dupefilters.
+* ``scrapy.contracts`` fixes and new features;
+* telnet console security improvements, first released as a
+ backport in :ref:`release-1.5.2`;
+* clean-up of the deprecated code;
+* various bug fixes, small new features and usability improvements across
+ the codebase.
+
+Selector API changes
+~~~~~~~~~~~~~~~~~~~~
+
+While these are not changes in Scrapy itself, but rather in the parsel_
+library which Scrapy uses for xpath/css selectors, these changes are
+worth mentioning here. Scrapy now depends on parsel >= 1.5, and
+Scrapy documentation is updated to follow recent ``parsel`` API conventions.
+
+Most visible change is that ``.get()`` and ``.getall()`` selector
+methods are now preferred over ``.extract_first()`` and ``.extract()``.
+We feel that these new methods result in a more concise and readable code.
+See :ref:`old-extraction-api` for more details.
+
+.. note::
+ There are currently **no plans** to deprecate ``.extract()``
+ and ``.extract_first()`` methods.
+
+Another useful new feature is the introduction of ``Selector.attrib`` and
+``SelectorList.attrib`` properties, which make it easier to get
+attributes of HTML elements. See :ref:`selecting-attributes`.
+
+CSS selectors are cached in parsel >= 1.5, which makes them faster
+when the same CSS path is used many times. This is very common in
+case of Scrapy spiders: callbacks are usually called several times,
+on different pages.
+
+If you're using custom ``Selector`` or ``SelectorList`` subclasses,
+a **backward incompatible** change in parsel may affect your code.
+See `parsel changelog`_ for a detailed description, as well as for the
+full list of improvements.
+
+.. _parsel changelog: https://parsel.readthedocs.io/en/latest/history.html
+
+Telnet console
+~~~~~~~~~~~~~~
+
+**Backward incompatible**: Scrapy's telnet console now requires username
+and password. See :ref:`topics-telnetconsole` for more details. This change
+fixes a **security issue**; see :ref:`release-1.5.2` release notes for details.
+
+New extensibility features
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* ``from_crawler`` support is added to feed exporters and feed storages. This,
+ among other things, allows to access Scrapy settings from custom feed
+ storages and exporters (:issue:`1605`, :issue:`3348`).
+* ``from_crawler`` support is added to dupefilters (:issue:`2956`); this allows
+ to access e.g. settings or a spider from a dupefilter.
+* :signal:`item_error` is fired when an error happens in a pipeline
+ (:issue:`3256`);
+* :signal:`request_reached_downloader` is fired when Downloader gets
+ a new Request; this signal can be useful e.g. for custom Schedulers
+ (:issue:`3393`).
+* new SitemapSpider :meth:`~.SitemapSpider.sitemap_filter` method which allows
+ to select sitemap entries based on their attributes in SitemapSpider
+ subclasses (:issue:`3512`).
+* Lazy loading of Downloader Handlers is now optional; this enables better
+ initialization error handling in custom Downloader Handlers (:issue:`3394`).
+
+New FilePipeline and MediaPipeline features
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Expose more options for S3FilesStore: :setting:`AWS_ENDPOINT_URL`,
+ :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`, :setting:`AWS_REGION_NAME`.
+ For example, this allows to use alternative or self-hosted
+ AWS-compatible providers (:issue:`2609`, :issue:`3548`).
+* ACL support for Google Cloud Storage: :setting:`FILES_STORE_GCS_ACL` and
+ :setting:`IMAGES_STORE_GCS_ACL` (:issue:`3199`).
+
+``scrapy.contracts`` improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Exceptions in contracts code are handled better (:issue:`3377`);
+* ``dont_filter=True`` is used for contract requests, which allows to test
+ different callbacks with the same URL (:issue:`3381`);
+* ``request_cls`` attribute in Contract subclasses allow to use different
+ Request classes in contracts, for example FormRequest (:issue:`3383`).
+* Fixed errback handling in contracts, e.g. for cases where a contract
+ is executed for URL which returns non-200 response (:issue:`3371`).
+
+Usability improvements
+~~~~~~~~~~~~~~~~~~~~~~
+
+* more stats for RobotsTxtMiddleware (:issue:`3100`)
+* INFO log level is used to show telnet host/port (:issue:`3115`)
+* a message is added to IgnoreRequest in RobotsTxtMiddleware (:issue:`3113`)
+* better validation of ``url`` argument in ``Response.follow`` (:issue:`3131`)
+* non-zero exit code is returned from Scrapy commands when error happens
+ on spider initialization (:issue:`3226`)
+* Link extraction improvements: "ftp" is added to scheme list (:issue:`3152`);
+ "flv" is added to common video extensions (:issue:`3165`)
+* better error message when an exporter is disabled (:issue:`3358`);
+* ``scrapy shell --help`` mentions syntax required for local files
+ (``./file.html``) - :issue:`3496`.
+* Referer header value is added to RFPDupeFilter log messages (:issue:`3588`)
+
+Bug fixes
+~~~~~~~~~
+
+* fixed issue with extra blank lines in .csv exports under Windows
+ (:issue:`3039`);
+* proper handling of pickling errors in Python 3 when serializing objects
+ for disk queues (:issue:`3082`)
+* flags are now preserved when copying Requests (:issue:`3342`);
+* FormRequest.from_response clickdata shouldn't ignore elements with
+ ``input[type=image]`` (:issue:`3153`).
+* FormRequest.from_response should preserve duplicate keys (:issue:`3247`)
+
+Documentation improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Docs are re-written to suggest .get/.getall API instead of
+ .extract/.extract_first. Also, :ref:`topics-selectors` docs are updated
+ and re-structured to match latest parsel docs; they now contain more topics,
+ such as :ref:`selecting-attributes` or :ref:`topics-selectors-css-extensions`
+ (:issue:`3390`).
+* :ref:`topics-developer-tools` is a new tutorial which replaces
+ old Firefox and Firebug tutorials (:issue:`3400`).
+* SCRAPY_PROJECT environment variable is documented (:issue:`3518`);
+* troubleshooting section is added to install instructions (:issue:`3517`);
+* improved links to beginner resources in the tutorial
+ (:issue:`3367`, :issue:`3468`);
+* fixed :setting:`RETRY_HTTP_CODES` default values in docs (:issue:`3335`);
+* remove unused ``DEPTH_STATS`` option from docs (:issue:`3245`);
+* other cleanups (:issue:`3347`, :issue:`3350`, :issue:`3445`, :issue:`3544`,
+ :issue:`3605`).
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+Compatibility shims for pre-1.0 Scrapy module names are removed
+(:issue:`3318`):
+
+* ``scrapy.command``
+* ``scrapy.contrib`` (with all submodules)
+* ``scrapy.contrib_exp`` (with all submodules)
+* ``scrapy.dupefilter``
+* ``scrapy.linkextractor``
+* ``scrapy.project``
+* ``scrapy.spider``
+* ``scrapy.spidermanager``
+* ``scrapy.squeue``
+* ``scrapy.stats``
+* ``scrapy.statscol``
+* ``scrapy.utils.decorator``
+
+See :ref:`module-relocations` for more information, or use suggestions
+from Scrapy 1.5.x deprecation warnings to update your code.
+
+Other deprecation removals:
+
+* Deprecated scrapy.interfaces.ISpiderManager is removed; please use
+ scrapy.interfaces.ISpiderLoader.
+* Deprecated ``CrawlerSettings`` class is removed (:issue:`3327`).
+* Deprecated ``Settings.overrides`` and ``Settings.defaults`` attributes
+ are removed (:issue:`3327`, :issue:`3359`).
+
+Other improvements, cleanups
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* All Scrapy tests now pass on Windows; Scrapy testing suite is executed
+ in a Windows environment on CI (:issue:`3315`).
+* Python 3.7 support (:issue:`3326`, :issue:`3150`, :issue:`3547`).
+* Testing and CI fixes (:issue:`3526`, :issue:`3538`, :issue:`3308`,
+ :issue:`3311`, :issue:`3309`, :issue:`3305`, :issue:`3210`, :issue:`3299`)
+* ``scrapy.http.cookies.CookieJar.clear`` accepts "domain", "path" and "name"
+ optional arguments (:issue:`3231`).
+* additional files are included to sdist (:issue:`3495`);
+* code style fixes (:issue:`3405`, :issue:`3304`);
+* unneeded .strip() call is removed (:issue:`3519`);
+* collections.deque is used to store MiddlewareManager methods instead
+ of a list (:issue:`3476`)
+
+.. _release-1.5.2:
+
+Scrapy 1.5.2 (2019-01-22)
+-------------------------
+
+* *Security bugfix*: Telnet console extension can be easily exploited by rogue
+ websites POSTing content to http://localhost:6023, we haven't found a way to
+ exploit it from Scrapy, but it is very easy to trick a browser to do so and
+ elevates the risk for local development environment.
+
+ *The fix is backward incompatible*, it enables telnet user-password
+ authentication by default with a random generated password. If you can't
+ upgrade right away, please consider setting :setting:`TELNETCONSOLE_PORT`
+ out of its default value.
+
+ See :ref:`telnet console ` documentation for more info
+
+* Backport CI build failure under GCE environment due to boto import error.
+
+.. _release-1.5.1:
+
+Scrapy 1.5.1 (2018-07-12)
+-------------------------
+
+This is a maintenance release with important bug fixes, but no new features:
+
+* ``O(N^2)`` gzip decompression issue which affected Python 3 and PyPy
+ is fixed (:issue:`3281`);
+* skipping of TLS validation errors is improved (:issue:`3166`);
+* Ctrl-C handling is fixed in Python 3.5+ (:issue:`3096`);
+* testing fixes (:issue:`3092`, :issue:`3263`);
+* documentation improvements (:issue:`3058`, :issue:`3059`, :issue:`3089`,
+ :issue:`3123`, :issue:`3127`, :issue:`3189`, :issue:`3224`, :issue:`3280`,
+ :issue:`3279`, :issue:`3201`, :issue:`3260`, :issue:`3284`, :issue:`3298`,
+ :issue:`3294`).
+
+
+.. _release-1.5.0:
+
Scrapy 1.5.0 (2017-12-29)
-------------------------
@@ -21,15 +820,15 @@ Some highlights:
* Better default handling of HTTP 308, 522 and 524 status codes.
* Documentation is improved, as usual.
-Backwards Incompatible Changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Backward Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* Scrapy 1.5 drops support for Python 3.3.
* Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
- **This is technically backwards-incompatible**; override
+ **This is technically backward-incompatible**; override
:setting:`USER_AGENT` if you relied on old value.
* Logging of settings overridden by ``custom_settings`` is fixed;
- **this is technically backwards-incompatible** because the logger
+ **this is technically backward-incompatible** because the logger
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``. If you're
parsing Scrapy logs, please update your log parsers (:issue:`1343`).
* LinkExtractor now ignores ``m4v`` extension by default, this is change
@@ -66,11 +865,11 @@ Bug fixes
~~~~~~~~~
- Fix logging of settings overridden by ``custom_settings``;
- **this is technically backwards-incompatible** because the logger
+ **this is technically backward-incompatible** because the logger
changes from ``[scrapy.utils.log]`` to ``[scrapy.crawler]``, so please
update your log parsers if needed (:issue:`1343`)
- Default Scrapy User-Agent now uses https link to scrapy.org (:issue:`2983`).
- **This is technically backwards-incompatible**; override
+ **This is technically backward-incompatible**; override
:setting:`USER_AGENT` if you relied on old value.
- Fix PyPy and PyPy3 test failures, support them officially
(:issue:`2793`, :issue:`2935`, :issue:`2990`, :issue:`3050`, :issue:`2213`,
@@ -114,6 +913,7 @@ Docs
- Document ``from_crawler`` methods for spider and downloader middlewares
(:issue:`3019`)
+.. _release-1.4.0:
Scrapy 1.4.0 (2017-05-18)
-------------------------
@@ -156,7 +956,7 @@ attributes with ``FormRequest``.
**Please also note that link extractors do not canonicalize URLs by default
anymore.** This was puzzling users every now and then, and it's not what
-browsers do in fact, so we removed that extra transformation on extractred
+browsers do in fact, so we removed that extra transformation on extracted
links.
For those of you wanting more control on the ``Referer:`` header that Scrapy
@@ -179,18 +979,20 @@ offset, using the new :setting:`FEED_EXPORT_INDENT` setting.
Enjoy! (Or read on for the rest of changes in this release.)
-Deprecations and Backwards Incompatible Changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Deprecations and Backward Incompatible Changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- Default to ``canonicalize=False`` in :class:`scrapy.linkextractors.LinkExtractor`
+- Default to ``canonicalize=False`` in
+ :class:`scrapy.linkextractors.LinkExtractor
+ `
(:issue:`2537`, fixes :issue:`1941` and :issue:`1982`):
- **warning, this is technically backwards-incompatible**
+ **warning, this is technically backward-incompatible**
- Enable memusage extension by default (:issue:`2539`, fixes :issue:`2187`);
- **this is technically backwards-incompatible** so please check if you have
+ **this is technically backward-incompatible** so please check if you have
any non-default ``MEMUSAGE_***`` options set.
- ``EDITOR`` environment variable now takes precedence over ``EDITOR``
option defined in settings.py (:issue:`1829`); Scrapy default settings
- no longer depend on environment variables. **This is technically a backwards
+ no longer depend on environment variables. **This is technically a backward
incompatible change**.
- ``Spider.make_requests_from_url`` is deprecated
(:issue:`1728`, fixes :issue:`1495`).
@@ -218,10 +1020,13 @@ New Features
- New ``data:`` URI download handler (:issue:`2334`, fixes :issue:`2156`)
- Log cache directory when HTTP Cache is used (:issue:`2611`, fixes :issue:`2604`)
- Warn users when project contains duplicate spider names (fixes :issue:`2181`)
-- :class:`CaselessDict` now accepts ``Mapping`` instances and not only dicts (:issue:`2646`)
-- :ref:`Media downloads `, with :class:`FilesPipelines`
- or :class:`ImagesPipelines`, can now optionally handle HTTP redirects
- using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting (:issue:`2616`, fixes :issue:`2004`)
+- ``scrapy.utils.datatypes.CaselessDict`` now accepts ``Mapping`` instances and
+ not only dicts (:issue:`2646`)
+- :ref:`Media downloads `, with
+ :class:`~scrapy.pipelines.files.FilesPipeline` or
+ :class:`~scrapy.pipelines.images.ImagesPipeline`, can now optionally handle
+ HTTP redirects using the new :setting:`MEDIA_ALLOW_REDIRECTS` setting
+ (:issue:`2616`, fixes :issue:`2004`)
- Accept non-complete responses from websites using a new
:setting:`DOWNLOAD_FAIL_ON_DATALOSS` setting (:issue:`2590`, fixes :issue:`2586`)
- Optional pretty-printing of JSON and XML items via
@@ -241,8 +1046,8 @@ Bug fixes
- LinkExtractor now strips leading and trailing whitespaces from attributes
(:issue:`2547`, fixes :issue:`1614`)
-- Properly handle whitespaces in action attribute in :class:`FormRequest`
- (:issue:`2548`)
+- Properly handle whitespaces in action attribute in
+ :class:`~scrapy.http.FormRequest` (:issue:`2548`)
- Buffer CONNECT response bytes from proxy until all HTTP headers are received
(:issue:`2495`, fixes :issue:`2491`)
- FTP downloader now works on Python 3, provided you use Twisted>=17.1
@@ -275,7 +1080,8 @@ Cleanups & Refactoring
fixes :issue:`2560`)
- Add omitted ``self`` arguments in default project middleware template (:issue:`2595`)
- Remove redundant ``slot.add_request()`` call in ExecutionEngine (:issue:`2617`)
-- Catch more specific ``os.error`` exception in :class:`FSFilesStore` (:issue:`2644`)
+- Catch more specific ``os.error`` exception in
+ ``scrapy.pipelines.files.FSFilesStore`` (:issue:`2644`)
- Change "localhost" test server certificate (:issue:`2720`)
- Remove unused ``MEMUSAGE_REPORT`` setting (:issue:`2576`)
@@ -292,7 +1098,8 @@ Documentation
(:issue:`2477`, fixes :issue:`2475`)
- FAQ: rewrite note on Python 3 support on Windows (:issue:`2690`)
- Rearrange selector sections (:issue:`2705`)
-- Remove ``__nonzero__`` from :class:`SelectorList` docs (:issue:`2683`)
+- Remove ``__nonzero__`` from :class:`~scrapy.selector.SelectorList`
+ docs (:issue:`2683`)
- Mention how to disable request filtering in documentation of
:setting:`DUPEFILTER_CLASS` setting (:issue:`2714`)
- Add sphinx_rtd_theme to docs setup readme (:issue:`2668`)
@@ -300,6 +1107,8 @@ Documentation
- Clarify ``allowed_domains`` example (:issue:`2670`)
+.. _release-1.3.3:
+
Scrapy 1.3.3 (2017-03-10)
-------------------------
@@ -312,6 +1121,7 @@ Bug fixes
A new setting is introduced to toggle between warning or exception if needed ;
see :setting:`SPIDER_LOADER_WARN_ONLY` for details.
+.. _release-1.3.2:
Scrapy 1.3.2 (2017-02-13)
-------------------------
@@ -323,6 +1133,8 @@ Bug fixes
- Use consistent selectors for author field in tutorial (:issue:`2551`).
- Fix TLS compatibility in Twisted 17+ (:issue:`2558`)
+.. _release-1.3.1:
+
Scrapy 1.3.1 (2017-02-08)
-------------------------
@@ -344,7 +1156,7 @@ Bug fixes
- Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`).
- Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`).
- Fix name of generated pipeline class when using basic project template (:issue:`2466`).
-- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`).
+- Fix compatibility with Twisted 17+ (:issue:`2496`, :issue:`2528`).
- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
@@ -352,7 +1164,7 @@ Bug fixes
Documentation
~~~~~~~~~~~~~
-- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4
+- Reword Code of Conduct section and upgrade to Contributor Covenant v1.4
(:issue:`2469`).
- Clarify that passing spider arguments converts them to spider attributes
(:issue:`2483`).
@@ -366,11 +1178,13 @@ Documentation
Cleanups
~~~~~~~~
-- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
+- Remove redundant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
- Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`).
- Remove dead code supporting old Twisted versions (:issue:`2544`).
+.. _release-1.3.0:
+
Scrapy 1.3.0 (2016-12-21)
-------------------------
@@ -390,13 +1204,13 @@ New Features
- ``MailSender`` now accepts single strings as values for ``to`` and ``cc``
arguments (:issue:`2272`)
- ``scrapy fetch url``, ``scrapy shell url`` and ``fetch(url)`` inside
- scrapy shell now follow HTTP redirections by default (:issue:`2290`);
+ Scrapy shell now follow HTTP redirections by default (:issue:`2290`);
See :command:`fetch` and :command:`shell` for details.
- ``HttpErrorMiddleware`` now logs errors with ``INFO`` level instead of ``DEBUG``;
- this is technically **backwards incompatible** so please check your log parsers.
+ this is technically **backward incompatible** so please check your log parsers.
- By default, logger names now use a long-form path, e.g. ``[scrapy.extensions.logstats]``,
instead of the shorter "top-level" variant of prior releases (e.g. ``[scrapy]``);
- this is **backwards incompatible** if you have log parsers expecting the short
+ this is **backward incompatible** if you have log parsers expecting the short
logger name part. You can switch back to short logger names using :setting:`LOG_SHORT_NAMES`
set to ``True``.
@@ -410,6 +1224,7 @@ Dependencies & Cleanups
- ``ChunkedTransferMiddleware`` is deprecated and removed from the default
downloader middlewares.
+.. _release-1.2.3:
Scrapy 1.2.3 (2017-03-03)
-------------------------
@@ -417,6 +1232,8 @@ Scrapy 1.2.3 (2017-03-03)
- Packaging fix: disallow unsupported Twisted versions in setup.py
+.. _release-1.2.2:
+
Scrapy 1.2.2 (2016-12-06)
-------------------------
@@ -452,6 +1269,8 @@ Other changes
.. _conda-forge: https://anaconda.org/conda-forge/scrapy
+.. _release-1.2.1:
+
Scrapy 1.2.1 (2016-10-21)
-------------------------
@@ -476,6 +1295,8 @@ Other changes
- Removed ``www.`` from ``start_urls`` in built-in spider templates (:issue:`2299`).
+.. _release-1.2.0:
+
Scrapy 1.2.0 (2016-10-03)
-------------------------
@@ -500,11 +1321,11 @@ Bug fixes
~~~~~~~~~
- DefaultRequestHeaders middleware now runs before UserAgent middleware
- (:issue:`2088`). **Warning: this is technically backwards incompatible**,
+ (:issue:`2088`). **Warning: this is technically backward incompatible**,
though we consider this a bug fix.
- HTTP cache extension and plugins that use the ``.scrapy`` data directory now
work outside projects (:issue:`1581`). **Warning: this is technically
- backwards incompatible**, though we consider this a bug fix.
+ backward incompatible**, though we consider this a bug fix.
- ``Selector`` does not allow passing both ``response`` and ``text`` anymore
(:issue:`2153`).
- Fixed logging of wrong callback name with ``scrapy parse`` (:issue:`2169`).
@@ -539,17 +1360,19 @@ Documentation
- Grammar fixes: :issue:`2128`, :issue:`1566`.
- Download stats badge removed from README (:issue:`2160`).
-- New scrapy :ref:`architecture diagram ` (:issue:`2165`).
+- New Scrapy :ref:`architecture diagram ` (:issue:`2165`).
- Updated ``Response`` parameters documentation (:issue:`2197`).
- Reworded misleading :setting:`RANDOMIZE_DOWNLOAD_DELAY` description (:issue:`2190`).
- Add StackOverflow as a support channel (:issue:`2257`).
+.. _release-1.1.4:
Scrapy 1.1.4 (2017-03-03)
-------------------------
- Packaging fix: disallow unsupported Twisted versions in setup.py
+.. _release-1.1.3:
Scrapy 1.1.3 (2016-09-22)
-------------------------
@@ -567,6 +1390,7 @@ Documentation
rewritten to use http://toscrape.com websites
(:issue:`2236`, :issue:`2249`, :issue:`2252`).
+.. _release-1.1.2:
Scrapy 1.1.2 (2016-08-18)
-------------------------
@@ -581,6 +1405,7 @@ Bug fixes
- :setting:`IMAGES_EXPIRES` default value set back to 90
(the regression was introduced in 1.1.1)
+.. _release-1.1.1:
Scrapy 1.1.1 (2016-07-13)
-------------------------
@@ -625,7 +1450,7 @@ Documentation
- Use "url" variable in downloader middleware example (:issue:`2015`)
- Grammar fixes (:issue:`2054`, :issue:`2120`)
- New FAQ entry on using BeautifulSoup in spider callbacks (:issue:`2048`)
-- Add notes about scrapy not working on Windows with Python 3 (:issue:`2060`)
+- Add notes about Scrapy not working on Windows with Python 3 (:issue:`2060`)
- Encourage complete titles in pull requests (:issue:`2026`)
Tests
@@ -633,6 +1458,7 @@ Tests
- Upgrade py.test requirement on Travis CI and Pin pytest-cov to 2.2.1 (:issue:`2095`)
+.. _release-1.1.0:
Scrapy 1.1.0 (2016-05-11)
-------------------------
@@ -674,18 +1500,18 @@ This 1.1 release brings a lot of interesting features and bug fixes:
this behavior, update :setting:`ROBOTSTXT_OBEY` in ``settings.py`` file
after creating a new project.
- Exporters now work on unicode, instead of bytes by default (:issue:`1080`).
- If you use ``PythonItemExporter``, you may want to update your code to
- disable binary mode which is now deprecated.
+ If you use :class:`~scrapy.exporters.PythonItemExporter`, you may want to
+ update your code to disable binary mode which is now deprecated.
- Accept XML node names containing dots as valid (:issue:`1533`).
- When uploading files or images to S3 (with ``FilesPipeline`` or
``ImagesPipeline``), the default ACL policy is now "private" instead
- of "public" **Warning: backwards incompatible!**.
+ of "public" **Warning: backward incompatible!**.
You can use :setting:`FILES_STORE_S3_ACL` to change it.
- We've reimplemented ``canonicalize_url()`` for more correct output,
especially for URLs with non-ASCII characters (:issue:`1947`).
- This could change link extractors output compared to previous scrapy versions.
+ This could change link extractors output compared to previous Scrapy versions.
This may also invalidate some cache entries you could still have from pre-1.1 runs.
- **Warning: backwards incompatible!**.
+ **Warning: backward incompatible!**.
Keep reading for more details on other improvements and bug fixes.
@@ -718,7 +1544,7 @@ Additional New Features and Enhancements
- Support for bpython and configure preferred Python shell via
``SCRAPY_PYTHON_SHELL`` (:issue:`1100`, :issue:`1444`).
- Support URLs without scheme (:issue:`1498`)
- **Warning: backwards incompatible!**
+ **Warning: backward incompatible!**
- Bring back support for relative file path (:issue:`1710`, :issue:`1550`).
- Added :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS` setting to change default check
@@ -801,7 +1627,7 @@ Bugfixes
~~~~~~~~
- Scrapy does not retry requests that got a ``HTTP 400 Bad Request``
- response anymore (:issue:`1289`). **Warning: backwards incompatible!**
+ response anymore (:issue:`1289`). **Warning: backward incompatible!**
- Support empty password for http_proxy config (:issue:`1274`).
- Interpret ``application/x-json`` as ``TextResponse`` (:issue:`1333`).
- Support link rel attribute with multiple values (:issue:`1201`).
@@ -813,8 +1639,8 @@ Bugfixes
- Fixed bug on ``XMLItemExporter`` with non-string fields in
items (:issue:`1738`).
- Fixed startproject command in OS X (:issue:`1635`).
-- Fixed PythonItemExporter and CSVExporter for non-string item
- types (:issue:`1737`).
+- Fixed :class:`~scrapy.exporters.PythonItemExporter` and CSVExporter for
+ non-string item types (:issue:`1737`).
- Various logging related fixes (:issue:`1294`, :issue:`1419`, :issue:`1263`,
:issue:`1624`, :issue:`1654`, :issue:`1722`, :issue:`1726` and :issue:`1303`).
- Fixed bug in ``utils.template.render_templatefile()`` (:issue:`1212`).
@@ -822,12 +1648,14 @@ Bugfixes
- HTTPS+CONNECT tunnels could get mixed up when using multiple proxies
to same remote host (:issue:`1912`).
+.. _release-1.0.7:
Scrapy 1.0.7 (2017-03-03)
-------------------------
- Packaging fix: disallow unsupported Twisted versions in setup.py
+.. _release-1.0.6:
Scrapy 1.0.6 (2016-05-04)
-------------------------
@@ -837,6 +1665,7 @@ Scrapy 1.0.6 (2016-05-04)
- DOC: Support for Sphinx 1.4+ (:issue:`1893`)
- DOC: Consistency in selectors examples (:issue:`1869`)
+.. _release-1.0.5:
Scrapy 1.0.5 (2016-02-04)
-------------------------
@@ -846,6 +1675,7 @@ Scrapy 1.0.5 (2016-02-04)
- DOC: Fixed typos in tutorial and media-pipeline (:commit:`808a9ea` and :commit:`803bd87`)
- DOC: Add AjaxCrawlMiddleware to DOWNLOADER_MIDDLEWARES_BASE in settings docs (:commit:`aa94121`)
+.. _release-1.0.4:
Scrapy 1.0.4 (2015-12-30)
-------------------------
@@ -875,7 +1705,7 @@ Scrapy 1.0.4 (2015-12-30)
- fix ValueError: Invalid XPath: //div/[id="not-exists"]/text() on selectors.rst (:commit:`ca8d60f`)
- Typos corrections (:commit:`7067117`)
- fix typos in downloader-middleware.rst and exceptions.rst, middlware -> middleware (:commit:`32f115c`)
-- Add note to ubuntu install section about debian compatibility (:commit:`23fda69`)
+- Add note to Ubuntu install section about Debian compatibility (:commit:`23fda69`)
- Replace alternative OSX install workaround with virtualenv (:commit:`98b63ee`)
- Reference Homebrew's homepage for installation instructions (:commit:`1925db1`)
- Add oldest supported tox version to contributing docs (:commit:`5d10d6d`)
@@ -892,19 +1722,23 @@ Scrapy 1.0.4 (2015-12-30)
- Merge pull request #1513 from mgedmin/patch-2 (:commit:`5d4daf8`)
- Typo (:commit:`f8d0682`)
- Fix list formatting (:commit:`5f83a93`)
-- fix scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`)
+- fix Scrapy squeue tests after recent changes to queuelib (:commit:`3365c01`)
- Merge pull request #1475 from rweindl/patch-1 (:commit:`2d688cd`)
- Update tutorial.rst (:commit:`fbc1f25`)
- Merge pull request #1449 from rhoekman/patch-1 (:commit:`7d6538c`)
- Small grammatical change (:commit:`8752294`)
- Add openssl version to version command (:commit:`13c45ac`)
+.. _release-1.0.3:
+
Scrapy 1.0.3 (2015-08-11)
-------------------------
-- add service_identity to scrapy install_requires (:commit:`cbc2501`)
+- add service_identity to Scrapy install_requires (:commit:`cbc2501`)
- Workaround for travis#296 (:commit:`66af9cd`)
+.. _release-1.0.2:
+
Scrapy 1.0.2 (2015-08-06)
-------------------------
@@ -915,6 +1749,8 @@ Scrapy 1.0.2 (2015-08-06)
- Fixed typos (:commit:`a9ae7b0`)
- Fix doc reference. (:commit:`7c8a4fe`)
+.. _release-1.0.1:
+
Scrapy 1.0.1 (2015-07-01)
-------------------------
@@ -922,9 +1758,11 @@ Scrapy 1.0.1 (2015-07-01)
- include tests/ to source distribution in MANIFEST.in (:commit:`eca227e`)
- DOC Fix SelectJmes documentation (:commit:`b8567bc`)
- DOC Bring Ubuntu and Archlinux outside of Windows subsection (:commit:`392233f`)
-- DOC remove version suffix from ubuntu package (:commit:`5303c66`)
+- DOC remove version suffix from Ubuntu package (:commit:`5303c66`)
- DOC Update release date for 1.0 (:commit:`c89fa29`)
+.. _release-1.0.0:
+
Scrapy 1.0.0 (2015-06-19)
-------------------------
@@ -1039,12 +1877,14 @@ until it reaches a stable status.
See more examples for scripts running Scrapy: :ref:`topics-practices`
+.. _module-relocations:
+
Module Relocations
~~~~~~~~~~~~~~~~~~
There’s been a large rearrangement of modules trying to improve the general
structure of Scrapy. Main changes were separating various subpackages into
-new projects and dissolving both `scrapy.contrib` and `scrapy.contrib_exp`
+new projects and dissolving both ``scrapy.contrib`` and ``scrapy.contrib_exp``
into top level packages. Backward compatibility was kept among internal
relocations, while importing deprecated modules expect warnings indicating
their new place.
@@ -1075,7 +1915,7 @@ Outsourced packages
| | /scrapy-plugins/scrapy-jsonrpc>`_ |
+-------------------------------------+-------------------------------------+
-`scrapy.contrib_exp` and `scrapy.contrib` dissolutions
+``scrapy.contrib_exp`` and ``scrapy.contrib`` dissolutions
+-------------------------------------+-------------------------------------+
| Old location | New location |
@@ -1287,7 +2127,7 @@ Code refactoring
(:issue:`1078`)
- Pydispatch pep8 (:issue:`992`)
- Removed unused 'load=False' parameter from walk_modules() (:issue:`871`)
-- For consistency, use `job_dir` helper in `SpiderState` extension.
+- For consistency, use ``job_dir`` helper in ``SpiderState`` extension.
(:issue:`805`)
- rename "sflo" local variables to less cryptic "log_observer" (:issue:`775`)
@@ -1371,13 +2211,13 @@ Scrapy 0.24.2 (2014-07-08)
- Use a mutable mapping to proxy deprecated settings.overrides and settings.defaults attribute (:commit:`e5e8133`)
- there is not support for python3 yet (:commit:`3cd6146`)
-- Update python compatible version set to debian packages (:commit:`fa5d76b`)
+- Update python compatible version set to Debian packages (:commit:`fa5d76b`)
- DOC fix formatting in release notes (:commit:`c6a9e20`)
Scrapy 0.24.1 (2014-06-27)
--------------------------
-- Fix deprecated CrawlerSettings and increase backwards compatibility with
+- Fix deprecated CrawlerSettings and increase backward compatibility with
.defaults attribute (:commit:`8e3f20a`)
@@ -1389,21 +2229,21 @@ Enhancements
- Improve Scrapy top-level namespace (:issue:`494`, :issue:`684`)
- Add selector shortcuts to responses (:issue:`554`, :issue:`690`)
-- Add new lxml based LinkExtractor to replace unmantained SgmlLinkExtractor
+- Add new lxml based LinkExtractor to replace unmaintained SgmlLinkExtractor
(:issue:`559`, :issue:`761`, :issue:`763`)
- Cleanup settings API - part of per-spider settings **GSoC project** (:issue:`737`)
- Add UTF8 encoding header to templates (:issue:`688`, :issue:`762`)
- Telnet console now binds to 127.0.0.1 by default (:issue:`699`)
-- Update debian/ubuntu install instructions (:issue:`509`, :issue:`549`)
+- Update Debian/Ubuntu install instructions (:issue:`509`, :issue:`549`)
- Disable smart strings in lxml XPath evaluations (:issue:`535`)
- Restore filesystem based cache as default for http
cache middleware (:issue:`541`, :issue:`500`, :issue:`571`)
- Expose current crawler in Scrapy shell (:issue:`557`)
- Improve testsuite comparing CSV and XML exporters (:issue:`570`)
-- New `offsite/filtered` and `offsite/domains` stats (:issue:`566`)
+- New ``offsite/filtered`` and ``offsite/domains`` stats (:issue:`566`)
- Support process_links as generator in CrawlSpider (:issue:`555`)
- Verbose logging and new stats counters for DupeFilter (:issue:`553`)
-- Add a mimetype parameter to `MailSender.send()` (:issue:`602`)
+- Add a mimetype parameter to ``MailSender.send()`` (:issue:`602`)
- Generalize file pipeline log messages (:issue:`622`)
- Replace unencodeable codepoints with html entities in SGMLLinkExtractor (:issue:`565`)
- Converted SEP documents to rst format (:issue:`629`, :issue:`630`,
@@ -1422,21 +2262,21 @@ Enhancements
- Make scrapy.version_info a tuple of integers (:issue:`681`, :issue:`692`)
- Infer exporter's output format from filename extensions
(:issue:`546`, :issue:`659`, :issue:`760`)
-- Support case-insensitive domains in `url_is_from_any_domain()` (:issue:`693`)
+- Support case-insensitive domains in ``url_is_from_any_domain()`` (:issue:`693`)
- Remove pep8 warnings in project and spider templates (:issue:`698`)
-- Tests and docs for `request_fingerprint` function (:issue:`597`)
-- Update SEP-19 for GSoC project `per-spider settings` (:issue:`705`)
+- Tests and docs for ``request_fingerprint`` function (:issue:`597`)
+- Update SEP-19 for GSoC project ``per-spider settings`` (:issue:`705`)
- Set exit code to non-zero when contracts fails (:issue:`727`)
-- Add a setting to control what class is instanciated as Downloader component
+- Add a setting to control what class is instantiated as Downloader component
(:issue:`738`)
-- Pass response in `item_dropped` signal (:issue:`724`)
-- Improve `scrapy check` contracts command (:issue:`733`, :issue:`752`)
-- Document `spider.closed()` shortcut (:issue:`719`)
-- Document `request_scheduled` signal (:issue:`746`)
+- Pass response in ``item_dropped`` signal (:issue:`724`)
+- Improve ``scrapy check`` contracts command (:issue:`733`, :issue:`752`)
+- Document ``spider.closed()`` shortcut (:issue:`719`)
+- Document ``request_scheduled`` signal (:issue:`746`)
- Add a note about reporting security issues (:issue:`697`)
- Add LevelDB http cache storage backend (:issue:`626`, :issue:`500`)
-- Sort spider list output of `scrapy list` command (:issue:`742`)
-- Multiple documentation enhancemens and fixes
+- Sort spider list output of ``scrapy list`` command (:issue:`742`)
+- Multiple documentation enhancements and fixes
(:issue:`575`, :issue:`587`, :issue:`590`, :issue:`596`, :issue:`610`,
:issue:`617`, :issue:`618`, :issue:`627`, :issue:`613`, :issue:`643`,
:issue:`654`, :issue:`675`, :issue:`663`, :issue:`711`, :issue:`714`)
@@ -1481,19 +2321,19 @@ Scrapy 0.22.1 (released 2014-02-08)
- BaseSgmlLinkExtractor: Added unit test of a link with an inner tag (:commit:`c1cb418`)
- BaseSgmlLinkExtractor: Fixed unknown_endtag() so that it only set current_link=None when the end tag match the opening tag (:commit:`7e4d627`)
- Fix tests for Travis-CI build (:commit:`76c7e20`)
-- replace unencodeable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
+- replace unencodable codepoints with html entities. fixes #562 and #285 (:commit:`5f87b17`)
- RegexLinkExtractor: encode URL unicode value when creating Links (:commit:`d0ee545`)
- Updated the tutorial crawl output with latest output. (:commit:`8da65de`)
- Updated shell docs with the crawler reference and fixed the actual shell output. (:commit:`875b9ab`)
- PEP8 minor edits. (:commit:`f89efaf`)
-- Expose current crawler in the scrapy shell. (:commit:`5349cec`)
+- Expose current crawler in the Scrapy shell. (:commit:`5349cec`)
- Unused re import and PEP8 minor edits. (:commit:`387f414`)
- Ignore None's values when using the ItemLoader. (:commit:`0632546`)
- DOC Fixed HTTPCACHE_STORAGE typo in the default value which is now Filesystem instead Dbm. (:commit:`cde9a8c`)
-- show ubuntu setup instructions as literal code (:commit:`fb5c9c5`)
+- show Ubuntu setup instructions as literal code (:commit:`fb5c9c5`)
- Update Ubuntu installation instructions (:commit:`70fb105`)
- Merge pull request #550 from stray-leone/patch-1 (:commit:`6f70b6a`)
-- modify the version of scrapy ubuntu package (:commit:`725900d`)
+- modify the version of Scrapy Ubuntu package (:commit:`725900d`)
- fix 0.22.0 release date (:commit:`af0219a`)
- fix typos in news.rst and remove (not released yet) header (:commit:`b7f58f4`)
@@ -1503,23 +2343,23 @@ Scrapy 0.22.0 (released 2014-01-17)
Enhancements
~~~~~~~~~~~~
-- [**Backwards incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
- To restore old backend set `HTTPCACHE_STORAGE` to `scrapy.contrib.httpcache.DbmCacheStorage`
+- [**Backward incompatible**] Switched HTTPCacheMiddleware backend to filesystem (:issue:`541`)
+ To restore old backend set ``HTTPCACHE_STORAGE`` to ``scrapy.contrib.httpcache.DbmCacheStorage``
- Proxy \https:// urls using CONNECT method (:issue:`392`, :issue:`397`)
- Add a middleware to crawl ajax crawleable pages as defined by google (:issue:`343`)
- Rename scrapy.spider.BaseSpider to scrapy.spider.Spider (:issue:`510`, :issue:`519`)
- Selectors register EXSLT namespaces by default (:issue:`472`)
- Unify item loaders similar to selectors renaming (:issue:`461`)
-- Make `RFPDupeFilter` class easily subclassable (:issue:`533`)
+- Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`)
- Improve test coverage and forthcoming Python 3 support (:issue:`525`)
- Promote startup info on settings and middleware to INFO level (:issue:`520`)
-- Support partials in `get_func_args` util (:issue:`506`, issue:`504`)
-- Allow running indiviual tests via tox (:issue:`503`)
+- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`)
+- Allow running individual tests via tox (:issue:`503`)
- Update extensions ignored by link extractors (:issue:`498`)
- Add middleware methods to get files/images/thumbs paths (:issue:`490`)
- Improve offsite middleware tests (:issue:`478`)
- Add a way to skip default Referer header set by RefererMiddleware (:issue:`475`)
-- Do not send `x-gzip` in default `Accept-Encoding` header (:issue:`469`)
+- Do not send ``x-gzip`` in default ``Accept-Encoding`` header (:issue:`469`)
- Support defining http error handling using settings (:issue:`466`)
- Use modern python idioms wherever you find legacies (:issue:`497`)
- Improve and correct documentation
@@ -1530,14 +2370,14 @@ Fixes
~~~~~
- Update Selector class imports in CrawlSpider template (:issue:`484`)
-- Fix unexistent reference to `engine.slots` (:issue:`464`)
-- Do not try to call `body_as_unicode()` on a non-TextResponse instance (:issue:`462`)
+- Fix unexistent reference to ``engine.slots`` (:issue:`464`)
+- Do not try to call ``body_as_unicode()`` on a non-TextResponse instance (:issue:`462`)
- Warn when subclassing XPathItemLoader, previously it only warned on
instantiation. (:issue:`523`)
- Warn when subclassing XPathSelector, previously it only warned on
instantiation. (:issue:`537`)
- Multiple fixes to memory stats (:issue:`531`, :issue:`530`, :issue:`529`)
-- Fix overriding url in `FormRequest.from_response()` (:issue:`507`)
+- Fix overriding url in ``FormRequest.from_response()`` (:issue:`507`)
- Fix tests runner under pip 1.5 (:issue:`513`)
- Fix logging error when spider name is unicode (:issue:`479`)
@@ -1564,24 +2404,24 @@ Enhancements
(modifying them had been deprecated for a long time)
- :setting:`ITEM_PIPELINES` is now defined as a dict (instead of a list)
- Sitemap spider can fetch alternate URLs (:issue:`360`)
-- `Selector.remove_namespaces()` now remove namespaces from element's attributes. (:issue:`416`)
+- ``Selector.remove_namespaces()`` now remove namespaces from element's attributes. (:issue:`416`)
- Paved the road for Python 3.3+ (:issue:`435`, :issue:`436`, :issue:`431`, :issue:`452`)
- New item exporter using native python types with nesting support (:issue:`366`)
- Tune HTTP1.1 pool size so it matches concurrency defined by settings (:commit:`b43b5f575`)
- scrapy.mail.MailSender now can connect over TLS or upgrade using STARTTLS (:issue:`327`)
- New FilesPipeline with functionality factored out from ImagesPipeline (:issue:`370`, :issue:`409`)
- Recommend Pillow instead of PIL for image handling (:issue:`317`)
-- Added debian packages for Ubuntu quantal and raring (:commit:`86230c0`)
+- Added Debian packages for Ubuntu Quantal and Raring (:commit:`86230c0`)
- Mock server (used for tests) can listen for HTTPS requests (:issue:`410`)
- Remove multi spider support from multiple core components
(:issue:`422`, :issue:`421`, :issue:`420`, :issue:`419`, :issue:`423`, :issue:`418`)
-- Travis-CI now tests Scrapy changes against development versions of `w3lib` and `queuelib` python packages.
+- Travis-CI now tests Scrapy changes against development versions of ``w3lib`` and ``queuelib`` python packages.
- Add pypy 2.1 to continuous integration tests (:commit:`ecfa7431`)
- Pylinted, pep8 and removed old-style exceptions from source (:issue:`430`, :issue:`432`)
- Use importlib for parametric imports (:issue:`445`)
- Handle a regression introduced in Python 2.7.5 that affects XmlItemExporter (:issue:`372`)
- Bugfix crawling shutdown on SIGINT (:issue:`450`)
-- Do not submit `reset` type inputs in FormRequest.from_response (:commit:`b326b87`)
+- Do not submit ``reset`` type inputs in FormRequest.from_response (:commit:`b326b87`)
- Do not silence download errors when request errback raises an exception (:commit:`684cfc0`)
Bugfixes
@@ -1590,14 +2430,14 @@ Bugfixes
- Fix tests under Django 1.6 (:commit:`b6bed44c`)
- Lot of bugfixes to retry middleware under disconnections using HTTP 1.1 download handler
- Fix inconsistencies among Twisted releases (:issue:`406`)
-- Fix scrapy shell bugs (:issue:`418`, :issue:`407`)
+- Fix Scrapy shell bugs (:issue:`418`, :issue:`407`)
- Fix invalid variable name in setup.py (:issue:`429`)
- Fix tutorial references (:issue:`387`)
- Improve request-response docs (:issue:`391`)
- Improve best practices docs (:issue:`399`, :issue:`400`, :issue:`401`, :issue:`402`)
- Improve django integration docs (:issue:`404`)
-- Document `bindaddress` request meta (:commit:`37c24e01d7`)
-- Improve `Request` class documentation (:issue:`226`)
+- Document ``bindaddress`` request meta (:commit:`37c24e01d7`)
+- Improve ``Request`` class documentation (:issue:`226`)
Other
~~~~~
@@ -1606,7 +2446,7 @@ Other
- Add `cssselect`_ python package as install dependency
- Drop libxml2 and multi selector's backend support, `lxml`_ is required from now on.
- Minimum Twisted version increased to 10.0.0, dropped Twisted 8.0 support.
-- Running test suite now requires `mock` python library (:issue:`390`)
+- Running test suite now requires ``mock`` python library (:issue:`390`)
Thanks
@@ -1660,7 +2500,7 @@ Scrapy 0.18.3 (released 2013-10-03)
Scrapy 0.18.2 (released 2013-09-03)
-----------------------------------
-- Backport `scrapy check` command fixes and backward compatible multi
+- Backport ``scrapy check`` command fixes and backward compatible multi
crawler process(:issue:`339`)
Scrapy 0.18.1 (released 2013-08-27)
@@ -1672,15 +2512,15 @@ Scrapy 0.18.1 (released 2013-08-27)
- test PotentiaDataLoss errors on unbound responses (:commit:`b15470d`)
- Treat responses without content-length or Transfer-Encoding as good responses (:commit:`c4bf324`)
- do no include ResponseFailed if http11 handler is not enabled (:commit:`6cbe684`)
-- New HTTP client wraps connection losts in ResponseFailed exception. fix #373 (:commit:`1a20bba`)
+- New HTTP client wraps connection lost in ResponseFailed exception. fix #373 (:commit:`1a20bba`)
- limit travis-ci build matrix (:commit:`3b01bb8`)
- Merge pull request #375 from peterarenot/patch-1 (:commit:`fa766d7`)
- Fixed so it refers to the correct folder (:commit:`3283809`)
-- added quantal & raring to support ubuntu releases (:commit:`1411923`)
+- added Quantal & Raring to support Ubuntu releases (:commit:`1411923`)
- fix retry middleware which didn't retry certain connection errors after the upgrade to http1 client, closes GH-373 (:commit:`bb35ed0`)
- fix XmlItemExporter in Python 2.7.4 and 2.7.5 (:commit:`de3e451`)
- minor updates to 0.18 release notes (:commit:`c45e5f1`)
-- fix contributters list format (:commit:`0b60031`)
+- fix contributors list format (:commit:`0b60031`)
Scrapy 0.18.0 (released 2013-08-09)
-----------------------------------
@@ -1689,36 +2529,36 @@ Scrapy 0.18.0 (released 2013-08-09)
- Handle GET parameters for AJAX crawleable urls (:commit:`3fe2a32`)
- Use lxml recover option to parse sitemaps (:issue:`347`)
- Bugfix cookie merging by hostname and not by netloc (:issue:`352`)
-- Support disabling `HttpCompressionMiddleware` using a flag setting (:issue:`359`)
-- Support xml namespaces using `iternodes` parser in `XMLFeedSpider` (:issue:`12`)
-- Support `dont_cache` request meta flag (:issue:`19`)
-- Bugfix `scrapy.utils.gz.gunzip` broken by changes in python 2.7.4 (:commit:`4dc76e`)
-- Bugfix url encoding on `SgmlLinkExtractor` (:issue:`24`)
-- Bugfix `TakeFirst` processor shouldn't discard zero (0) value (:issue:`59`)
+- Support disabling ``HttpCompressionMiddleware`` using a flag setting (:issue:`359`)
+- Support xml namespaces using ``iternodes`` parser in ``XMLFeedSpider`` (:issue:`12`)
+- Support ``dont_cache`` request meta flag (:issue:`19`)
+- Bugfix ``scrapy.utils.gz.gunzip`` broken by changes in python 2.7.4 (:commit:`4dc76e`)
+- Bugfix url encoding on ``SgmlLinkExtractor`` (:issue:`24`)
+- Bugfix ``TakeFirst`` processor shouldn't discard zero (0) value (:issue:`59`)
- Support nested items in xml exporter (:issue:`66`)
- Improve cookies handling performance (:issue:`77`)
- Log dupe filtered requests once (:issue:`105`)
- Split redirection middleware into status and meta based middlewares (:issue:`78`)
- Use HTTP1.1 as default downloader handler (:issue:`109` and :issue:`318`)
-- Support xpath form selection on `FormRequest.from_response` (:issue:`185`)
-- Bugfix unicode decoding error on `SgmlLinkExtractor` (:issue:`199`)
+- Support xpath form selection on ``FormRequest.from_response`` (:issue:`185`)
+- Bugfix unicode decoding error on ``SgmlLinkExtractor`` (:issue:`199`)
- Bugfix signal dispatching on pypi interpreter (:issue:`205`)
- Improve request delay and concurrency handling (:issue:`206`)
-- Add RFC2616 cache policy to `HttpCacheMiddleware` (:issue:`212`)
+- Add RFC2616 cache policy to ``HttpCacheMiddleware`` (:issue:`212`)
- Allow customization of messages logged by engine (:issue:`214`)
-- Multiples improvements to `DjangoItem` (:issue:`217`, :issue:`218`, :issue:`221`)
+- Multiples improvements to ``DjangoItem`` (:issue:`217`, :issue:`218`, :issue:`221`)
- Extend Scrapy commands using setuptools entry points (:issue:`260`)
-- Allow spider `allowed_domains` value to be set/tuple (:issue:`261`)
-- Support `settings.getdict` (:issue:`269`)
-- Simplify internal `scrapy.core.scraper` slot handling (:issue:`271`)
-- Added `Item.copy` (:issue:`290`)
+- Allow spider ``allowed_domains`` value to be set/tuple (:issue:`261`)
+- Support ``settings.getdict`` (:issue:`269`)
+- Simplify internal ``scrapy.core.scraper`` slot handling (:issue:`271`)
+- Added ``Item.copy`` (:issue:`290`)
- Collect idle downloader slots (:issue:`297`)
-- Add `ftp://` scheme downloader handler (:issue:`329`)
+- Add ``ftp://`` scheme downloader handler (:issue:`329`)
- Added downloader benchmark webserver and spider tools :ref:`benchmarking`
-- Moved persistent (on disk) queues to a separate project (queuelib_) which scrapy now depends on
-- Add scrapy commands using external libraries (:issue:`260`)
+- Moved persistent (on disk) queues to a separate project (queuelib_) which Scrapy now depends on
+- Add Scrapy commands using external libraries (:issue:`260`)
- Added ``--pdb`` option to ``scrapy`` command line tool
-- Added :meth:`XPathSelector.remove_namespaces` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
+- Added :meth:`XPathSelector.remove_namespaces ` which allows to remove all namespaces from XML documents for convenience (to work with namespace-less XPaths). Documented in :ref:`topics-selectors`.
- Several improvements to spider contracts
- New default middleware named MetaRefreshMiddldeware that handles meta-refresh html tag redirections,
- MetaRefreshMiddldeware and RedirectMiddleware have different priorities to address #62
@@ -1728,7 +2568,7 @@ Scrapy 0.18.0 (released 2013-08-09)
- several more cleanups to singletons and multi-spider support (thanks Nicolas Ramirez)
- support custom download slots
- added --spider option to "shell" command.
-- log overridden settings when scrapy starts
+- log overridden settings when Scrapy starts
Thanks to everyone who contribute to this release. Here is a list of
contributors sorted by number of commits::
@@ -1777,7 +2617,7 @@ contributors sorted by number of commits::
Scrapy 0.16.5 (released 2013-05-30)
-----------------------------------
-- obey request method when scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`)
+- obey request method when Scrapy deploy is redirected to a new endpoint (:commit:`8c4fcee`)
- fix inaccurate downloader middleware documentation. refs #280 (:commit:`40667cb`)
- doc: remove links to diveintopython.org, which is no longer available. closes #246 (:commit:`bd58bfa`)
- Find form nodes in invalid html5 documents (:commit:`e3d6945`)
@@ -1791,8 +2631,8 @@ Scrapy 0.16.4 (released 2013-01-23)
- Fixed error message formatting. log.err() doesn't support cool formatting and when error occurred, the message was: "ERROR: Error processing %(item)s" (:commit:`c16150c`)
- lint and improve images pipeline error logging (:commit:`56b45fc`)
- fixed doc typos (:commit:`243be84`)
-- add documentation topics: Broad Crawls & Common Practies (:commit:`1fbb715`)
-- fix bug in scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`)
+- add documentation topics: Broad Crawls & Common Practices (:commit:`1fbb715`)
+- fix bug in Scrapy parse command when spider is not specified explicitly. closes #209 (:commit:`c72e682`)
- Update docs/topics/commands.rst (:commit:`28eac7a`)
Scrapy 0.16.3 (released 2012-12-07)
@@ -1811,11 +2651,11 @@ Scrapy 0.16.3 (released 2012-12-07)
Scrapy 0.16.2 (released 2012-11-09)
-----------------------------------
-- scrapy contracts: python2.6 compat (:commit:`a4a9199`)
-- scrapy contracts verbose option (:commit:`ec41673`)
-- proper unittest-like output for scrapy contracts (:commit:`86635e4`)
+- Scrapy contracts: python2.6 compat (:commit:`a4a9199`)
+- Scrapy contracts verbose option (:commit:`ec41673`)
+- proper unittest-like output for Scrapy contracts (:commit:`86635e4`)
- added open_in_browser to debugging doc (:commit:`c9b690d`)
-- removed reference to global scrapy stats from settings doc (:commit:`dd55067`)
+- removed reference to global Scrapy stats from settings doc (:commit:`dd55067`)
- Fix SpiderState bug in Windows platforms (:commit:`58998f4`)
@@ -1823,9 +2663,9 @@ Scrapy 0.16.1 (released 2012-10-26)
-----------------------------------
- fixed LogStats extension, which got broken after a wrong merge before the 0.16 release (:commit:`8c780fd`)
-- better backwards compatibility for scrapy.conf.settings (:commit:`3403089`)
+- better backward compatibility for scrapy.conf.settings (:commit:`3403089`)
- extended documentation on how to access crawler stats from extensions (:commit:`c4da0b5`)
-- removed .hgtags (no longer needed now that scrapy uses git) (:commit:`d52c188`)
+- removed .hgtags (no longer needed now that Scrapy uses git) (:commit:`d52c188`)
- fix dashes under rst headers (:commit:`fa4f7f9`)
- set release date for 0.16.0 in news (:commit:`e292246`)
@@ -1838,13 +2678,12 @@ Scrapy changes:
- added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way
- added options ``-o`` and ``-t`` to the :command:`runspider` command
- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
-- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backwards compatibility is kept on the Stats Collector API and signals.
-- added :meth:`~scrapy.contrib.spidermiddleware.SpiderMiddleware.process_start_requests` method to spider middlewares
-- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
-- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info.
+- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
+- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares
+- dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info.
- dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info.
- documented :ref:`topics-api`
-- `lxml` is now the default selectors backend instead of `libxml2`
+- ``lxml`` is now the default selectors backend instead of ``libxml2``
- ported FormRequest.from_response() to use `lxml`_ instead of `ClientForm`_
- removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm``
- SitemapSpider: added support for sitemap urls ending in .xml and .xml.gz, even if they advertise a wrong content type (:commit:`10ed28b`)
@@ -1863,7 +2702,7 @@ Scrapy changes:
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
- promoted :ref:`topics-djangoitem` to main contrib
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
-- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
+- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- removed signal: ``scrapy.mail.mail_sent``
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled
@@ -1875,7 +2714,7 @@ Scrapy changes:
Scrapy 0.14.4
-------------
-- added precise to supported ubuntu distros (:commit:`b7e46df`)
+- added precise to supported Ubuntu distros (:commit:`b7e46df`)
- fixed bug in json-rpc webservice reported in https://groups.google.com/forum/#!topic/scrapy-users/qgVBmFybNAQ/discussion. also removed no longer supported 'run' command from extras/scrapy-ws.py (:commit:`340fbdb`)
- meta tag attributes for content-type http equiv can be in any order. #123 (:commit:`0cb68af`)
- replace "import Image" by more standard "from PIL import Image". closes #88 (:commit:`4d17048`)
@@ -1888,11 +2727,11 @@ Scrapy 0.14.3
- include egg files used by testsuite in source distribution. #118 (:commit:`c897793`)
- update docstring in project template to avoid confusion with genspider command, which may be considered as an advanced feature. refs #107 (:commit:`2548dcc`)
- added note to docs/topics/firebug.rst about google directory being shut down (:commit:`668e352`)
-- dont discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
+- don't discard slot when empty, just save in another dict in order to recycle if needed again. (:commit:`8e9f607`)
- do not fail handling unicode xpaths in libxml2 backed selectors (:commit:`b830e95`)
- fixed minor mistake in Request objects documentation (:commit:`bf3c9ee`)
- fixed minor defect in link extractors documentation (:commit:`ba14f38`)
-- removed some obsolete remaining code related to sqlite support in scrapy (:commit:`0665175`)
+- removed some obsolete remaining code related to sqlite support in Scrapy (:commit:`0665175`)
Scrapy 0.14.2
-------------
@@ -1937,16 +2776,16 @@ New features and settings
- New ``ChunkedTransferMiddleware`` (enabled by default) to support `chunked transfer encoding`_ (:rev:`2769`)
- Add boto 2.0 support for S3 downloader handler (:rev:`2763`)
- Added `marshal`_ to formats supported by feed exports (:rev:`2744`)
-- In request errbacks, offending requests are now received in `failure.request` attribute (:rev:`2738`)
+- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
- check the documentation for more details
- Added builtin caching DNS resolver (:rev:`2728`)
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
-- Moved spider queues to scrapyd: `scrapy.spiderqueue` -> `scrapyd.spiderqueue` (:rev:`2708`)
-- Moved sqlite utils to scrapyd: `scrapy.utils.sqlite` -> `scrapyd.sqlite` (:rev:`2781`)
-- Real support for returning iterators on `start_requests()` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`)
+- Moved spider queues to scrapyd: ``scrapy.spiderqueue`` -> ``scrapyd.spiderqueue`` (:rev:`2708`)
+- Moved sqlite utils to scrapyd: ``scrapy.utils.sqlite`` -> ``scrapyd.sqlite`` (:rev:`2781`)
+- Real support for returning iterators on ``start_requests()`` method. The iterator is now consumed during the crawl when the spider is getting idle (:rev:`2704`)
- Added :setting:`REDIRECT_ENABLED` setting to quickly enable/disable the redirect middleware (:rev:`2697`)
- Added :setting:`RETRY_ENABLED` setting to quickly enable/disable the retry middleware (:rev:`2694`)
- Added ``CloseSpider`` exception to manually close spiders (:rev:`2691`)
@@ -1954,19 +2793,19 @@ New features and settings
- Refactored close spider behavior to wait for all downloads to finish and be processed by spiders, before closing the spider (:rev:`2688`)
- Added ``SitemapSpider`` (see documentation in Spiders page) (:rev:`2658`)
- Added ``LogStats`` extension for periodically logging basic stats (like crawled pages and scraped items) (:rev:`2657`)
-- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an `IOError`.
+- Make handling of gzipped responses more robust (#319, :rev:`2643`). Now Scrapy will try and decompress as much as possible from a gzipped response, instead of failing with an ``IOError``.
- Simplified !MemoryDebugger extension to use stats for dumping memory debugging info (:rev:`2639`)
-- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and `-e` flag to `genspider` command that uses it (:rev:`2653`)
+- Added new command to edit spiders: ``scrapy edit`` (:rev:`2636`) and ``-e`` flag to ``genspider`` command that uses it (:rev:`2653`)
- Changed default representation of items to pretty-printed dicts. (:rev:`2631`). This improves default logging by making log more readable in the default case, for both Scraped and Dropped lines.
- Added :signal:`spider_error` signal (:rev:`2628`)
- Added :setting:`COOKIES_ENABLED` setting (:rev:`2625`)
-- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to `True`). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there.
+- Stats are now dumped to Scrapy log (default value of :setting:`STATS_DUMP` setting has been changed to ``True``). This is to make Scrapy users more aware of Scrapy stats and the data that is collected there.
- Added support for dynamically adjusting download delay and maximum concurrent requests (:rev:`2599`)
- Added new DBM HTTP cache storage backend (:rev:`2576`)
- Added ``listjobs.json`` API to Scrapyd (:rev:`2571`)
- ``CsvItemExporter``: added ``join_multivalued`` parameter (:rev:`2578`)
- Added namespace support to ``xmliter_lxml`` (:rev:`2552`)
-- Improved cookies middleware by making `COOKIES_DEBUG` nicer and documenting it (:rev:`2579`)
+- Improved cookies middleware by making ``COOKIES_DEBUG`` nicer and documenting it (:rev:`2579`)
- Several improvements to Scrapyd and Link extractors
Code rearranged and removed
@@ -1980,17 +2819,17 @@ Code rearranged and removed
- Reduced Scrapy codebase by striping part of Scrapy code into two new libraries:
- `w3lib`_ (several functions from ``scrapy.utils.{http,markup,multipart,response,url}``, done in :rev:`2584`)
- `scrapely`_ (was ``scrapy.contrib.ibl``, done in :rev:`2586`)
-- Removed unused function: `scrapy.utils.request.request_info()` (:rev:`2577`)
-- Removed googledir project from `examples/googledir`. There's now a new example project called `dirbot` available on github: https://github.com/scrapy/dirbot
+- Removed unused function: ``scrapy.utils.request.request_info()`` (:rev:`2577`)
+- Removed googledir project from ``examples/googledir``. There's now a new example project called ``dirbot`` available on GitHub: https://github.com/scrapy/dirbot
- Removed support for default field values in Scrapy items (:rev:`2616`)
- Removed experimental crawlspider v2 (:rev:`2632`)
-- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (`DUPEFILTER_CLASS` setting) (:rev:`2640`)
+- Removed scheduler middleware to simplify architecture. Duplicates filter is now done in the scheduler itself, using the same dupe fltering class as before (``DUPEFILTER_CLASS`` setting) (:rev:`2640`)
- Removed support for passing urls to ``scrapy crawl`` command (use ``scrapy parse`` instead) (:rev:`2704`)
- Removed deprecated Execution Queue (:rev:`2704`)
- Removed (undocumented) spider context extension (from scrapy.contrib.spidercontext) (:rev:`2780`)
- removed ``CONCURRENT_SPIDERS`` setting (use scrapyd maxproc instead) (:rev:`2789`)
- Renamed attributes of core components: downloader.sites -> downloader.slots, scraper.sites -> scraper.slots (:rev:`2717`, :rev:`2718`)
-- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backwards compatibility kept.
+- Renamed setting ``CLOSESPIDER_ITEMPASSED`` to :setting:`CLOSESPIDER_ITEMCOUNT` (:rev:`2655`). Backward compatibility kept.
Scrapy 0.12
-----------
@@ -2000,7 +2839,8 @@ The numbers like #NNN reference tickets in the old issue tracker (Trac) which is
New features and improvements
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- Passed item is now sent in the ``item`` argument of the :signal:`item_passed` (#273)
+- Passed item is now sent in the ``item`` argument of the :signal:`item_passed
+ ` (#273)
- Added verbose option to ``scrapy version`` command, useful for bug reports (#298)
- HTTP cache now stored by default in the project data dir (#279)
- Added project data storage directory (#276, #277)
@@ -2020,13 +2860,13 @@ Scrapyd changes
- Scrapyd now uses one process per spider
- It stores one log file per spider run, and rotate them keeping the lastest 5 logs per spider (by default)
- A minimal web ui was added, available at http://localhost:6800 by default
-- There is now a `scrapy server` command to start a Scrapyd server of the current project
+- There is now a ``scrapy server`` command to start a Scrapyd server of the current project
Changes to settings
~~~~~~~~~~~~~~~~~~~
-- added `HTTPCACHE_ENABLED` setting (False by default) to enable HTTP cache middleware
-- changed `HTTPCACHE_EXPIRATION_SECS` semantics: now zero means "never expire".
+- added ``HTTPCACHE_ENABLED`` setting (False by default) to enable HTTP cache middleware
+- changed ``HTTPCACHE_EXPIRATION_SECS`` semantics: now zero means "never expire".
Deprecated/obsoleted functionality
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2057,17 +2897,17 @@ New features and improvements
- Splitted Debian package into two packages - the library and the service (#187)
- Scrapy log refactoring (#188)
- New extension for keeping persistent spider contexts among different runs (#203)
-- Added `dont_redirect` request.meta key for avoiding redirects (#233)
-- Added `dont_retry` request.meta key for avoiding retries (#234)
+- Added ``dont_redirect`` request.meta key for avoiding redirects (#233)
+- Added ``dont_retry`` request.meta key for avoiding retries (#234)
Command-line tool changes
~~~~~~~~~~~~~~~~~~~~~~~~~
-- New `scrapy` command which replaces the old `scrapy-ctl.py` (#199)
- - there is only one global `scrapy` command now, instead of one `scrapy-ctl.py` per project
- - Added `scrapy.bat` script for running more conveniently from Windows
+- New ``scrapy`` command which replaces the old ``scrapy-ctl.py`` (#199)
+ - there is only one global ``scrapy`` command now, instead of one ``scrapy-ctl.py`` per project
+ - Added ``scrapy.bat`` script for running more conveniently from Windows
- Added bash completion to command-line tool (#210)
-- Renamed command `start` to `runserver` (#209)
+- Renamed command ``start`` to ``runserver`` (#209)
API changes
~~~~~~~~~~~
@@ -2076,7 +2916,7 @@ API changes
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
-- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
+- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
- Changes to Scrapy Manager (now called "Crawler"):
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
@@ -2087,11 +2927,11 @@ API changes
- ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector``
- default per-command settings are now specified in the ``default_settings`` attribute of command object class (#201)
- changed arguments of Item pipeline ``process_item()`` method from ``(spider, item)`` to ``(item, spider)``
- - backwards compatibility kept (with deprecation warning)
+ - backward compatibility kept (with deprecation warning)
- moved ``scrapy.core.signals`` module to ``scrapy.signals``
- - backwards compatibility kept (with deprecation warning)
+ - backward compatibility kept (with deprecation warning)
- moved ``scrapy.core.exceptions`` module to ``scrapy.exceptions``
- - backwards compatibility kept (with deprecation warning)
+ - backward compatibility kept (with deprecation warning)
- added ``handles_request()`` class method to ``BaseSpider``
- dropped ``scrapy.log.exc()`` function (use ``scrapy.log.err()`` instead)
- dropped ``component`` argument of ``scrapy.log.msg()`` function
@@ -2162,8 +3002,8 @@ New features
- Added support for HTTP proxies (``HttpProxyMiddleware``) (:rev:`1781`, :rev:`1785`)
- Offsite spider middleware now logs messages when filtering out requests (:rev:`1841`)
-Backwards-incompatible changes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Changed ``scrapy.utils.response.get_meta_refresh()`` signature (:rev:`1804`)
- Removed deprecated ``scrapy.item.ScrapedItem`` class - use ``scrapy.item.Item instead`` (:rev:`1838`)
@@ -2201,13 +3041,35 @@ First release of Scrapy.
.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
+.. _botocore: https://github.com/boto/botocore
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
-.. _w3lib: https://github.com/scrapy/w3lib
-.. _scrapely: https://github.com/scrapy/scrapely
-.. _marshal: https://docs.python.org/2/library/marshal.html
-.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
-.. _lxml: http://lxml.de/
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
-.. _resource: https://docs.python.org/2/library/resource.html
+.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _cryptography: https://cryptography.io/en/latest/
+.. _cssselect: https://github.com/scrapy/cssselect/
+.. _docstrings: https://docs.python.org/glossary.html#term-docstring
+.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
+.. _LevelDB: https://github.com/google/leveldb
+.. _lxml: http://lxml.de/
+.. _marshal: https://docs.python.org/2/library/marshal.html
+.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
+.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
+.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
+.. _Pillow: https://python-pillow.org/
+.. _pyOpenSSL: https://www.pyopenssl.org/en/stable/
.. _queuelib: https://github.com/scrapy/queuelib
-.. _cssselect: https://github.com/SimonSapin/cssselect
+.. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
+.. _resource: https://docs.python.org/2/library/resource.html
+.. _robots.txt: http://www.robotstxt.org/
+.. _scrapely: https://github.com/scrapy/scrapely
+.. _service_identity: https://service-identity.readthedocs.io/en/stable/
+.. _six: https://six.readthedocs.io/
+.. _tox: https://pypi.python.org/pypi/tox
+.. _Twisted: https://twistedmatrix.com/trac/
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
+.. _w3lib: https://github.com/scrapy/w3lib
+.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
+.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
+.. _Zsh: https://www.zsh.org/
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 8e7611d21..773b92cea 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,4 @@
-Sphinx>=1.6
-sphinx_rtd_theme
\ No newline at end of file
+Sphinx>=2.1
+sphinx-hoverxref
+sphinx-notfound-page
+sphinx_rtd_theme
diff --git a/docs/topics/_images/firebug1.png b/docs/topics/_images/firebug1.png
deleted file mode 100644
index e2eaefa83..000000000
Binary files a/docs/topics/_images/firebug1.png and /dev/null differ
diff --git a/docs/topics/_images/firebug2.png b/docs/topics/_images/firebug2.png
deleted file mode 100644
index 4cab63431..000000000
Binary files a/docs/topics/_images/firebug2.png and /dev/null differ
diff --git a/docs/topics/_images/firebug3.png b/docs/topics/_images/firebug3.png
deleted file mode 100644
index affbe14bc..000000000
Binary files a/docs/topics/_images/firebug3.png and /dev/null differ
diff --git a/docs/topics/_images/inspector_01.png b/docs/topics/_images/inspector_01.png
new file mode 100644
index 000000000..edb8795dc
Binary files /dev/null and b/docs/topics/_images/inspector_01.png differ
diff --git a/docs/topics/_images/network_01.png b/docs/topics/_images/network_01.png
new file mode 100644
index 000000000..1788ea76a
Binary files /dev/null and b/docs/topics/_images/network_01.png differ
diff --git a/docs/topics/_images/network_02.png b/docs/topics/_images/network_02.png
new file mode 100644
index 000000000..5d39ae601
Binary files /dev/null and b/docs/topics/_images/network_02.png differ
diff --git a/docs/topics/_images/network_03.png b/docs/topics/_images/network_03.png
new file mode 100644
index 000000000..472fca958
Binary files /dev/null and b/docs/topics/_images/network_03.png differ
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 985cc0433..1c461a511 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -94,11 +94,13 @@ how you :ref:`configure the downloader middlewares
.. method:: crawl(\*args, \**kwargs)
Starts the crawler by instantiating its spider class with the given
- `args` and `kwargs` arguments, while setting the execution engine in
+ ``args`` and ``kwargs`` arguments, while setting the execution engine in
motion.
Returns a deferred that is fired when the crawl is finished.
+ .. automethod:: stop
+
.. autoclass:: CrawlerRunner
:members:
@@ -154,7 +156,7 @@ Settings API
SpiderLoader API
================
-.. module:: scrapy.loader
+.. module:: scrapy.spiderloader
:synopsis: The spider loader
.. class:: SpiderLoader
@@ -180,7 +182,7 @@ SpiderLoader API
.. method:: load(spider_name)
Get the Spider class with the given name. It'll look into the previously
- loaded spiders for a spider class with name `spider_name` and will raise
+ loaded spiders for a spider class with name ``spider_name`` and will raise
a KeyError if not found.
:param spider_name: spider class name
@@ -271,5 +273,3 @@ class (which they all inherit from).
Close the given spider. After this is called, no more specific stats
can be accessed or collected.
-
-.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 4ac39ad2d..ae25dfa2f 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -166,11 +166,10 @@ for concurrency.
For more information about asynchronous programming and Twisted see these
links:
-* `Introduction to Deferreds in Twisted`_
+* :doc:`twisted:core/howto/defer-intro`
* `Twisted - hello, asynchronous programming`_
* `Twisted Introduction - Krondo`_
.. _Twisted: https://twistedmatrix.com/trac/
-.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
-.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index c9bece753..4317019fc 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -11,7 +11,7 @@ Design goals
============
1. be nicer to sites instead of using default download delay of zero
-2. automatically adjust scrapy to the optimum crawling speed, so the user
+2. automatically adjust Scrapy to the optimum crawling speed, so the user
doesn't have to tune the download delays to find the optimum one.
The user only needs to specify the maximum concurrent requests
it allows, and the extension does the rest.
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index eb02086dc..1ab08d949 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -39,24 +39,50 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
concrete suggestions of Scrapy settings to tune in order to achieve an
efficient broad crawl.
+.. _broad-crawls-scheduler-priority-queue:
+
+Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
+=================================================
+
+Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
+It works best during single-domain crawl. It does not work well with crawling
+many different domains in parallel
+
+To apply the recommended priority queue use::
+
+ SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+
+.. _broad-crawls-concurrency:
+
Increase concurrency
====================
Concurrency is the number of requests that are processed in parallel. There is
-a global limit and a per-domain limit.
+a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
+can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per
+IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
+
+.. note:: The scheduler priority queue :ref:`recommended for broad crawls
+ ` does not support
+ :setting:`CONCURRENT_REQUESTS_PER_IP`.
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much
-to increase it will depend on how much CPU you crawler will have available. A
-good starting point is ``100``, but the best way to find out is by doing some
-trials and identifying at what concurrency your Scrapy process gets CPU
-bounded. For optimum performance, you should pick a concurrency where CPU usage
-is at 80-90%.
+to increase it will depend on how much CPU and memory you crawler will have
+available.
-To increase the global concurrency use::
+A good starting point is ``100``::
CONCURRENT_REQUESTS = 100
+But the best way to find out is by doing some trials and identifying at what
+concurrency your Scrapy process gets CPU bounded. For optimum performance, you
+should pick a concurrency where CPU usage is at 80-90%.
+
+Increasing concurrency also increases memory usage. If memory usage is a
+concern, you might need to lower your global concurrency limit accordingly.
+
+
Increase Twisted IO thread pool maximum size
============================================
@@ -85,7 +111,7 @@ When doing broad crawls you are often only interested in the crawl rates you
get and any errors found. These stats are reported by Scrapy when using the
``INFO`` log level. In order to save CPU (and log storage requirements) you
should not use ``DEBUG`` log level when preforming large broad crawls in
-production. Using ``DEBUG`` level when developing your (broad) crawler may be
+production. Using ``DEBUG`` level when developing your (broad) crawler may be
fine though.
To set the log level use::
@@ -163,3 +189,25 @@ It is turned OFF by default because it has some performance overhead,
and enabling it for focused crawls doesn't make much sense.
.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+
+.. _broad-crawls-bfo:
+
+Crawl in BFO order
+==================
+
+:ref:`Scrapy crawls in DFO order by default `.
+
+In broad crawls, however, page crawling tends to be faster than page
+processing. As a result, unprocessed early requests stay in memory until the
+final depth is reached, which can significantly increase memory usage.
+
+:ref:`Crawl in BFO order ` instead to save memory.
+
+
+Be mindful of memory leaks
+==========================
+
+If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
+BFO order ` and :ref:`lowering concurrency
+` you should :ref:`debug your memory leaks
+`.
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 3088017cb..a0dcba90d 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -1,3 +1,5 @@
+.. highlight:: none
+
.. _topics-commands:
=================
@@ -27,7 +29,7 @@ in standard locations:
1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
for global (user-wide) settings, and
-3. ``scrapy.cfg`` inside a scrapy project's root (see next section).
+3. ``scrapy.cfg`` inside a Scrapy project's root (see next section).
Settings from these files are merged in the listed order of preference:
user-defined values have higher priority than system-wide defaults
@@ -37,7 +39,7 @@ Scrapy also understands, and can be configured through, a number of environment
variables. Currently these are:
* ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
-* ``SCRAPY_PROJECT``
+* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`)
* ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
.. _topics-project-structure:
@@ -66,11 +68,42 @@ structure by default, similar to this::
The directory where the ``scrapy.cfg`` file resides is known as the *project
root directory*. That file contains the name of the python module that defines
-the project settings. Here is an example::
+the project settings. Here is an example:
+
+.. code-block:: ini
[settings]
default = myproject.settings
+.. _topics-project-envvar:
+
+Sharing the root directory between projects
+===========================================
+
+A project root directory, the one that contains the ``scrapy.cfg``, may be
+shared by multiple Scrapy projects, each with its own settings module.
+
+In that case, you must define one or more aliases for those settings modules
+under ``[settings]`` in your ``scrapy.cfg`` file:
+
+.. code-block:: ini
+
+ [settings]
+ default = myproject1.settings
+ project1 = myproject1.settings
+ project2 = myproject2.settings
+
+By default, the ``scrapy`` command-line tool will use the ``default`` settings.
+Use the ``SCRAPY_PROJECT`` environment variable to specify a different project
+for ``scrapy`` to use::
+
+ $ scrapy settings --get BOT_NAME
+ Project 1 Bot
+ $ export SCRAPY_PROJECT=project2
+ $ scrapy settings --get BOT_NAME
+ Project 2 Bot
+
+
Using the ``scrapy`` tool
=========================
@@ -250,6 +283,8 @@ check
Run contract checks.
+.. skip: start
+
Usage examples::
$ scrapy check -l
@@ -267,6 +302,8 @@ Usage examples::
[FAILED] first_spider:parse
>>> Returned 92 requests, expected 0..4
+.. skip: end
+
.. command:: list
list
@@ -434,6 +471,9 @@ Supported options:
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
+* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
+ This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}'
+
* ``--pipelines``: process items through pipelines
* ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
@@ -451,6 +491,8 @@ Supported options:
* ``--verbose`` or ``-v``: display information for each depth level
+.. skip: start
+
Usage example::
$ scrapy parse http://www.example.com/ -c parse_item
@@ -458,13 +500,15 @@ Usage example::
>>> STATUS DEPTH LEVEL 1 <<<
# Scraped Items ------------------------------------------------------------
- [{'name': u'Example item',
- 'category': u'Furniture',
- 'length': u'12 cm'}]
+ [{'name': 'Example item',
+ 'category': 'Furniture',
+ 'length': '12 cm'}]
# Requests -----------------------------------------------------------------
[]
+.. skip: end
+
.. command:: settings
@@ -543,7 +587,9 @@ Default: ``''`` (empty string)
A module to use for looking up custom Scrapy commands. This is used to add custom
commands for your Scrapy project.
-Example::
+Example:
+
+.. code-block:: python
COMMANDS_MODULE = 'mybot.commands'
@@ -558,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
``scrapy.commands`` section in the entry points of the library ``setup.py``
file.
-The following example adds ``my_command`` command::
+The following example adds ``my_command`` command:
+
+.. skip: next
+
+.. code-block:: python
from setuptools import setup, find_packages
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index ba1421c42..43db8f101 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -6,10 +6,6 @@ Spiders Contracts
.. versionadded:: 0.15
-.. note:: This is a new feature (introduced in Scrapy 0.15) and may be subject
- to minor functionality/API updates. Check the :ref:`release notes ` to
- be notified of updates.
-
Testing spiders can get particularly annoying and while nothing prevents you
from writing unit tests the task gets cumbersome quickly. Scrapy offers an
integrated way of testing your spiders by the means of contracts.
@@ -35,12 +31,20 @@ This callback is tested using three built-in contracts:
.. class:: UrlContract
- This contract (``@url``) sets the sample url used when checking other
+ This contract (``@url``) sets the sample URL used when checking other
contract conditions for this spider. This contract is mandatory. All
callbacks lacking this contract are ignored when running the checks::
@url url
+.. class:: CallbackKeywordArgumentsContract
+
+ This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs `
+ attribute for the sample request. It must be a valid JSON dictionary.
+ ::
+
+ @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
+
.. class:: ReturnsContract
This contract (``@returns``) sets lower and upper bounds for the items and
@@ -60,7 +64,7 @@ Use the :command:`check` command to run the contract checks.
Custom Contracts
================
-If you find you need more power than the built-in scrapy contracts you can
+If you find you need more power than the built-in Scrapy contracts you can
create and load your own contracts in the project by using the
:setting:`SPIDER_CONTRACTS` setting::
@@ -69,7 +73,7 @@ create and load your own contracts in the project by using the
'myproject.contracts.ItemValidate': 10,
}
-Each contract must inherit from :class:`scrapy.contracts.Contract` and can
+Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
override three methods:
.. module:: scrapy.contracts
@@ -86,8 +90,11 @@ override three methods:
.. method:: Contract.adjust_request_args(args)
This receives a ``dict`` as an argument containing default arguments
- for :class:`~scrapy.http.Request` object. Must return the same or a
- modified version of it.
+ for request object. :class:`~scrapy.http.Request` is used by default,
+ but this can be changed with the ``request_cls`` attribute.
+ If multiple contracts in chain have this attribute defined, the last one is used.
+
+ Must return the same or a modified version of it.
.. method:: Contract.pre_process(response)
@@ -99,9 +106,14 @@ override three methods:
This allows processing the output of the callback. Iterators are
converted listified before being passed to this hook.
+Raise :class:`~scrapy.exceptions.ContractFail` from
+:class:`~scrapy.contracts.Contract.pre_process` or
+:class:`~scrapy.contracts.Contract.post_process` if expectations are not met:
+
+.. autoclass:: scrapy.exceptions.ContractFail
+
Here is a demo contract which checks the presence of a custom header in the
-response received. Raise :class:`scrapy.exceptions.ContractFail` in order to
-get the failures pretty printed::
+response received::
from scrapy.contracts import Contract
from scrapy.exceptions import ContractFail
@@ -117,3 +129,24 @@ get the failures pretty printed::
for header in self.args:
if header not in response.headers:
raise ContractFail('X-CustomHeader not present')
+
+.. _detecting-contract-check-runs:
+
+Detecting check runs
+====================
+
+When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
+set to the ``true`` string. You can use `os.environ`_ to perform any change to
+your spiders or your settings when ``scrapy check`` is used::
+
+ import os
+ import scrapy
+
+ class ExampleSpider(scrapy.Spider):
+ name = 'example'
+
+ def __init__(self):
+ if os.environ.get('SCRAPY_CHECK'):
+ pass # Do some scraper adjustments when a check is running
+
+.. _os.environ: https://docs.python.org/3/library/os.html#os.environ
diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index d1991c02f..d75f17301 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -5,7 +5,7 @@ Debugging Spiders
=================
This document explains the most common techniques for debugging spiders.
-Consider the following scrapy spider below::
+Consider the following Scrapy spider below::
import scrapy
from myproject.items import MyItem
@@ -18,24 +18,25 @@ Consider the following scrapy spider below::
)
def parse(self, response):
- # collect `item_urls`
+ #
+ # collect `item_urls`
for item_url in item_urls:
yield scrapy.Request(item_url, self.parse_item)
def parse_item(self, response):
+ #
item = MyItem()
# populate `item` fields
# and extract item_details_url
- yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
+ yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
- def parse_details(self, response):
- item = response.meta['item']
+ def parse_details(self, response, item):
# populate more `item` fields
return item
Basically this is a simple spider which parses two pages of items (the
start_urls). Items also have a details page with additional information, so we
-use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a
+use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
partially populated item.
@@ -47,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
of the spider at the method level. It has the advantage of being flexible and
simple to use, but does not allow debugging code inside a method.
+.. highlight:: none
+
+.. skip: start
+
In order to see the item scraped from a specific url::
$ scrapy parse --spider=myspider -c parse_item -d 2
@@ -84,6 +89,8 @@ using::
$ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
+.. skip: end
+
Scrapy Shell
============
@@ -93,13 +100,14 @@ spider, it is of little help to check what happens inside a callback, besides
showing the response received and the output. How to debug the situation when
``parse_details`` sometimes receives no item?
+.. highlight:: python
+
Fortunately, the :command:`shell` is your bread and butter in this case (see
:ref:`topics-shell-inspect-response`)::
from scrapy.shell import inspect_response
- def parse_details(self, response):
- item = response.meta.get('item', None)
+ def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
@@ -132,8 +140,7 @@ Logging is another useful option for getting information about your spider run.
Although not as convenient, it comes with the advantage that the logs will be
available in all future runs should they be necessary again::
- def parse_details(self, response):
- item = response.meta.get('item', None)
+ def parse_details(self, response, item=None):
if item:
# populate more `item` fields
return item
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
new file mode 100644
index 000000000..b7d740c19
--- /dev/null
+++ b/docs/topics/developer-tools.rst
@@ -0,0 +1,306 @@
+.. _topics-developer-tools:
+
+=================================================
+Using your browser's Developer Tools for scraping
+=================================================
+
+Here is a general guide on how to use your browser's Developer Tools
+to ease the scraping process. Today almost all browsers come with
+built in `Developer Tools`_ and although we will use Firefox in this
+guide, the concepts are applicable to any other browser.
+
+In this guide we'll introduce the basic tools to use from a browser's
+Developer Tools by scraping `quotes.toscrape.com`_.
+
+.. _topics-livedom:
+
+Caveats with inspecting the live browser DOM
+============================================
+
+Since Developer Tools operate on a live browser DOM, what you'll actually see
+when inspecting the page source is not the original HTML, but a modified one
+after applying some browser clean up and executing Javascript code. Firefox,
+in particular, is known for adding ```` elements to tables. Scrapy, on
+the other hand, does not modify the original page HTML, so you won't be able to
+extract any data if you use `` `` in your XPath expressions.
+
+Therefore, you should keep in mind the following things:
+
+* Disable Javascript while inspecting the DOM looking for XPaths to be
+ used in Scrapy (in the Developer Tools settings click `Disable JavaScript`)
+
+* Never use full XPath paths, use relative and clever ones based on attributes
+ (such as ``id``, ``class``, ``width``, etc) or any identifying features like
+ ``contains(@href, 'image')``.
+
+* Never include `` `` elements in your XPath expressions unless you
+ really know what you're doing
+
+.. _topics-inspector:
+
+Inspecting a website
+====================
+
+By far the most handy feature of the Developer Tools is the `Inspector`
+feature, which allows you to inspect the underlying HTML code of
+any webpage. To demonstrate the Inspector, let's look at the
+`quotes.toscrape.com`_-site.
+
+On the site we have a total of ten quotes from various authors with specific
+tags, as well as the Top Ten Tags. Let's say we want to extract all the quotes
+on this page, without any meta-information about authors, tags, etc.
+
+Instead of viewing the whole source code for the page, we can simply right click
+on a quote and select ``Inspect Element (Q)``, which opens up the `Inspector`.
+In it you should see something like this:
+
+.. image:: _images/inspector_01.png
+ :width: 777
+ :height: 469
+ :alt: Firefox's Inspector-tool
+
+The interesting part for us is this:
+
+.. code-block:: html
+
+
+
(...)
+
(...)
+
(...)
+
+
+ “The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+
+
(...)
+
(...)
+
` to
+follow the pagination. On a simple site such as this, there may not be
+the need to find an element visually but the ``Scroll into View`` function
+can be quite useful on complex sites.
+
+Note that the search bar can also be used to search for and test CSS
+selectors. For example, you could search for ``span.text`` to find
+all quote texts. Instead of a full text search, this searches for
+exactly the ``span`` tag with the ``class="text"`` in the page.
+
+.. _topics-network-tool:
+
+The Network-tool
+================
+While scraping you may come across dynamic webpages where some parts
+of the page are loaded dynamically through multiple requests. While
+this can be quite tricky, the `Network`-tool in the Developer Tools
+greatly facilitates this task. To demonstrate the Network-tool, let's
+take a look at the page `quotes.toscrape.com/scroll`_.
+
+The page is quite similar to the basic `quotes.toscrape.com`_-page,
+but instead of the above-mentioned ``Next`` button, the page
+automatically loads new quotes when you scroll to the bottom. We
+could go ahead and try out different XPaths directly, but instead
+we'll check another quite useful command from the Scrapy shell:
+
+.. skip: next
+
+.. code-block:: none
+
+ $ scrapy shell "quotes.toscrape.com/scroll"
+ (...)
+ >>> view(response)
+
+A browser window should open with the webpage but with one
+crucial difference: Instead of the quotes we just see a greenish
+bar with the word ``Loading...``.
+
+.. image:: _images/network_01.png
+ :width: 777
+ :height: 296
+ :alt: Response from quotes.toscrape.com/scroll
+
+The ``view(response)`` command let's us view the response our
+shell or later our spider receives from the server. Here we see
+that some basic template is loaded which includes the title,
+the login-button and the footer, but the quotes are missing. This
+tells us that the quotes are being loaded from a different request
+than ``quotes.toscrape/scroll``.
+
+If you click on the ``Network`` tab, you will probably only see
+two entries. The first thing we do is enable persistent logs by
+clicking on ``Persist Logs``. If this option is disabled, the
+log is automatically cleared each time you navigate to a different
+page. Enabling this option is a good default, since it gives us
+control on when to clear the logs.
+
+If we reload the page now, you'll see the log get populated with six
+new requests.
+
+.. image:: _images/network_02.png
+ :width: 777
+ :height: 241
+ :alt: Network tab with persistent logs and requests
+
+Here we see every request that has been made when reloading the page
+and can inspect each request and its response. So let's find out
+where our quotes are coming from:
+
+First click on the request with the name ``scroll``. On the right
+you can now inspect the request. In ``Headers`` you'll find details
+about the request headers, such as the URL, the method, the IP-address,
+and so on. We'll ignore the other tabs and click directly on ``Response``.
+
+What you should see in the ``Preview`` pane is the rendered HTML-code,
+that is exactly what we saw when we called ``view(response)`` in the
+shell. Accordingly the ``type`` of the request in the log is ``html``.
+The other requests have types like ``css`` or ``js``, but what
+interests us is the one request called ``quotes?page=1`` with the
+type ``json``.
+
+If we click on this request, we see that the request URL is
+``http://quotes.toscrape.com/api/quotes?page=1`` and the response
+is a JSON-object that contains our quotes. We can also right-click
+on the request and open ``Open in new tab`` to get a better overview.
+
+.. image:: _images/network_03.png
+ :width: 777
+ :height: 375
+ :alt: JSON-object returned from the quotes.toscrape API
+
+With this response we can now easily parse the JSON-object and
+also request each page to get every quote on the site::
+
+ import scrapy
+ import json
+
+
+ class QuoteSpider(scrapy.Spider):
+ name = 'quote'
+ allowed_domains = ['quotes.toscrape.com']
+ page = 1
+ start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
+
+ def parse(self, response):
+ data = json.loads(response.text)
+ for quote in data["quotes"]:
+ yield {"quote": quote["text"]}
+ if data["has_next"]:
+ self.page += 1
+ url = "http://quotes.toscrape.com/api/quotes?page={}".format(self.page)
+ yield scrapy.Request(url=url, callback=self.parse)
+
+This spider starts at the first page of the quotes-API. With each
+response, we parse the ``response.text`` and assign it to ``data``.
+This lets us operate on the JSON-object like on a Python dictionary.
+We iterate through the ``quotes`` and print out the ``quote["text"]``.
+If the handy ``has_next`` element is ``true`` (try loading
+`quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
+page-number greater than 10), we increment the ``page`` attribute
+and ``yield`` a new request, inserting the incremented page-number
+into our ``url``.
+
+.. _requests-from-curl:
+
+In more complex websites, it could be difficult to easily reproduce the
+requests, as we could need to add ``headers`` or ``cookies`` to make it work.
+In those cases you can export the requests in `cURL `.
.. module:: scrapy.downloadermiddlewares
@@ -193,7 +199,7 @@ CookiesMiddleware
This middleware enables working with sites that require cookies, such as
those that use sessions. It keeps track of cookies sent by web servers, and
- send them back on subsequent requests (from that spider), just like web
+ sends them back on subsequent requests (from that spider), just like web
browsers do.
The following settings can be used to configure the cookie middleware:
@@ -342,10 +348,9 @@ HttpCacheMiddleware
* :ref:`httpcache-storage-fs`
* :ref:`httpcache-storage-dbm`
- * :ref:`httpcache-storage-leveldb`
You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
- setting. Or you can also implement your own storage backend.
+ setting. Or you can also :ref:`implement your own storage backend. `
Scrapy ships with two HTTP cache policies:
@@ -357,26 +362,27 @@ HttpCacheMiddleware
.. reqmeta:: dont_cache
- You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`.
+ You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``.
+
+.. module:: scrapy.extensions.httpcache
+ :noindex:
.. _httpcache-policy-dummy:
Dummy policy (default)
~~~~~~~~~~~~~~~~~~~~~~
-This policy has no awareness of any HTTP Cache-Control directives.
-Every request and its corresponding response are cached. When the same
-request is seen again, the response is returned without transferring
-anything from the Internet.
+.. class:: DummyPolicy
-The Dummy policy is useful for testing spiders faster (without having
-to wait for downloads every time) and for trying your spider offline,
-when an Internet connection is not available. The goal is to be able to
-"replay" a spider run *exactly as it ran before*.
+ This policy has no awareness of any HTTP Cache-Control directives.
+ Every request and its corresponding response are cached. When the same
+ request is seen again, the response is returned without transferring
+ anything from the Internet.
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy``
+ The Dummy policy is useful for testing spiders faster (without having
+ to wait for downloads every time) and for trying your spider offline,
+ when an Internet connection is not available. The goal is to be able to
+ "replay" a spider run *exactly as it ran before*.
.. _httpcache-policy-rfc2616:
@@ -384,45 +390,44 @@ In order to use this policy, set:
RFC2616 policy
~~~~~~~~~~~~~~
-This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
-Cache-Control awareness, aimed at production and used in continuous
-runs to avoid downloading unmodified data (to save bandwidth and speed up crawls).
+.. class:: RFC2616Policy
-what is implemented:
+ This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
+ Cache-Control awareness, aimed at production and used in continuous
+ runs to avoid downloading unmodified data (to save bandwidth and speed up
+ crawls).
-* Do not attempt to store responses/requests with `no-store` cache-control directive set
-* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses
-* Compute freshness lifetime from `max-age` cache-control directive
-* Compute freshness lifetime from `Expires` response header
-* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox)
-* Compute current age from `Age` response header
-* Compute current age from `Date` header
-* Revalidate stale responses based on `Last-Modified` response header
-* Revalidate stale responses based on `ETag` response header
-* Set `Date` header for any received response missing it
-* Support `max-stale` cache-control directive in requests
+ What is implemented:
- This allows spiders to be configured with the full RFC2616 cache policy,
- but avoid revalidation on a request-by-request basis, while remaining
- conformant with the HTTP spec.
+ * Do not attempt to store responses/requests with ``no-store`` cache-control directive set
+ * Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses
+ * Compute freshness lifetime from ``max-age`` cache-control directive
+ * Compute freshness lifetime from ``Expires`` response header
+ * Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox)
+ * Compute current age from ``Age`` response header
+ * Compute current age from ``Date`` header
+ * Revalidate stale responses based on ``Last-Modified`` response header
+ * Revalidate stale responses based on ``ETag`` response header
+ * Set ``Date`` header for any received response missing it
+ * Support ``max-stale`` cache-control directive in requests
- Example:
+ This allows spiders to be configured with the full RFC2616 cache policy,
+ but avoid revalidation on a request-by-request basis, while remaining
+ conformant with the HTTP spec.
- Add `Cache-Control: max-stale=600` to Request headers to accept responses that
- have exceeded their expiration time by no more than 600 seconds.
+ Example:
- See also: RFC2616, 14.9.3
+ Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that
+ have exceeded their expiration time by no more than 600 seconds.
-what is missing:
+ See also: RFC2616, 14.9.3
-* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
-* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
-* Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
-* ... probably others ..
+ What is missing:
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy``
+ * ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+ * ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
+ * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
+ * ... probably others ..
.. _httpcache-storage-fs:
@@ -430,67 +435,102 @@ In order to use this policy, set:
Filesystem storage backend (default)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-File system storage backend is available for the HTTP cache middleware.
+.. class:: FilesystemCacheStorage
-In order to use this storage backend, set:
+ File system storage backend is available for the HTTP cache middleware.
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage``
+ Each request/response pair is stored in a different directory containing
+ the following files:
-Each request/response pair is stored in a different directory containing
-the following files:
+ * ``request_body`` - the plain request body
- * ``request_body`` - the plain request body
- * ``request_headers`` - the request headers (in raw HTTP format)
- * ``response_body`` - the plain response body
- * ``response_headers`` - the request headers (in raw HTTP format)
- * ``meta`` - some metadata of this cache resource in Python ``repr()`` format
- (grep-friendly format)
- * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
- efficient deserialization
+ * ``request_headers`` - the request headers (in raw HTTP format)
-The directory name is made from the request fingerprint (see
-``scrapy.utils.request.fingerprint``), and one level of subdirectories is
-used to avoid creating too many files into the same directory (which is
-inefficient in many file systems). An example directory could be::
+ * ``response_body`` - the plain response body
- /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
+ * ``response_headers`` - the request headers (in raw HTTP format)
+
+ * ``meta`` - some metadata of this cache resource in Python ``repr()``
+ format (grep-friendly format)
+
+ * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
+ efficient deserialization
+
+ The directory name is made from the request fingerprint (see
+ ``scrapy.utils.request.fingerprint``), and one level of subdirectories is
+ used to avoid creating too many files into the same directory (which is
+ inefficient in many file systems). An example directory could be::
+
+ /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
.. _httpcache-storage-dbm:
DBM storage backend
~~~~~~~~~~~~~~~~~~~
-.. versionadded:: 0.13
+.. class:: DbmCacheStorage
-A DBM_ storage backend is also available for the HTTP cache middleware.
+ .. versionadded:: 0.13
-By default, it uses the anydbm_ module, but you can change it with the
-:setting:`HTTPCACHE_DBM_MODULE` setting.
+ A DBM_ storage backend is also available for the HTTP cache middleware.
-In order to use this storage backend, set:
+ By default, it uses the :mod:`dbm`, but you can change it with the
+ :setting:`HTTPCACHE_DBM_MODULE` setting.
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage``
+.. _httpcache-storage-custom:
-.. _httpcache-storage-leveldb:
+Writing your own storage backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-LevelDB storage backend
-~~~~~~~~~~~~~~~~~~~~~~~
+You can implement a cache storage backend by creating a Python class that
+defines the methods described below.
-.. versionadded:: 0.23
+.. module:: scrapy.extensions.httpcache
-A LevelDB_ storage backend is also available for the HTTP cache middleware.
+.. class:: CacheStorage
-This backend is not recommended for development because only one process can
-access LevelDB databases at the same time, so you can't run a crawl and open
-the scrapy shell in parallel for the same spider.
+ .. method:: open_spider(spider)
-In order to use this storage backend:
+ This method gets called after a spider has been opened for crawling. It handles
+ the :signal:`open_spider ` signal.
-* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage``
-* install `LevelDB python bindings`_ like ``pip install leveldb``
+ :param spider: the spider which has been opened
+ :type spider: :class:`~scrapy.spiders.Spider` object
-.. _LevelDB: https://github.com/google/leveldb
-.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
+ .. method:: close_spider(spider)
+
+ This method gets called after a spider has been closed. It handles
+ the :signal:`close_spider ` signal.
+
+ :param spider: the spider which has been closed
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
+ .. method:: retrieve_response(spider, request)
+
+ Return response if present in cache, or ``None`` otherwise.
+
+ :param spider: the spider which generated the request
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
+ :param request: the request to find cached response for
+ :type request: :class:`~scrapy.http.Request` object
+
+ .. method:: store_response(spider, request, response)
+
+ Store the given response in the cache.
+
+ :param spider: the spider for which the response is intended
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
+ :param request: the corresponding request the spider generated
+ :type request: :class:`~scrapy.http.Request` object
+
+ :param response: the response to store in the cache
+ :type response: :class:`~scrapy.http.Response` object
+
+In order to use your storage backend, set:
+
+* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class.
HTTPCache middleware settings
@@ -586,7 +626,7 @@ HTTPCACHE_DBM_MODULE
.. versionadded:: 0.13
-Default: ``'anydbm'``
+Default: ``'dbm'``
The database module to use in the :ref:`DBM storage backend
`. This setting is specific to the DBM backend.
@@ -626,13 +666,13 @@ Default: ``False``
If enabled, will cache pages unconditionally.
A spider may wish to have all responses available in the cache, for
-future use with `Cache-Control: max-stale`, for instance. The
+future use with ``Cache-Control: max-stale``, for instance. The
DummyPolicy caches all responses but never revalidates them, and
sometimes a more nuanced policy is desirable.
-This setting still respects `Cache-Control: no-store` directives in responses.
-If you don't want that, filter `no-store` out of the Cache-Control headers in
-responses you feedto the cache middleware.
+This setting still respects ``Cache-Control: no-store`` directives in responses.
+If you don't want that, filter ``no-store`` out of the Cache-Control headers in
+responses you feed to the cache middleware.
.. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
@@ -646,7 +686,7 @@ Default: ``[]``
List of Cache-Control directives in responses to be ignored.
Sites often set "no-store", "no-cache", "must-revalidate", etc., but get
-upset at the traffic a spider can generate if it respects those
+upset at the traffic a spider can generate if it actually respects those
directives. This allows to selectively ignore Cache-Control directives
that are known to be unimportant for the sites being crawled.
@@ -729,6 +769,17 @@ RedirectMiddleware
The urls which the request goes through (while being redirected) can be found
in the ``redirect_urls`` :attr:`Request.meta ` key.
+.. reqmeta:: redirect_reasons
+
+The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
+``redirect_reasons`` :attr:`Request.meta ` key. For
+example: ``[301, 302, 307, 'meta refresh']``.
+
+The format of a reason depends on the middleware that handled the corresponding
+redirect. For example, :class:`RedirectMiddleware` indicates the triggering
+response status code as an integer, while :class:`MetaRefreshMiddleware`
+always uses the ``'meta refresh'`` string as reason.
+
The :class:`RedirectMiddleware` can be configured through the following
settings (see the settings documentation for more info):
@@ -790,10 +841,12 @@ The :class:`MetaRefreshMiddleware` can be configured through the following
settings (see the settings documentation for more info):
* :setting:`METAREFRESH_ENABLED`
+* :setting:`METAREFRESH_IGNORE_TAGS`
* :setting:`METAREFRESH_MAXDELAY`
-This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`
-and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`
+This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
+:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described
+for :class:`RedirectMiddleware`
MetaRefreshMiddleware settings
@@ -810,6 +863,15 @@ Default: ``True``
Whether the Meta Refresh middleware will be enabled.
+.. setting:: METAREFRESH_IGNORE_TAGS
+
+METAREFRESH_IGNORE_TAGS
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``['script', 'noscript']``
+
+Meta tags within these tags are ignored.
+
.. setting:: METAREFRESH_MAXDELAY
METAREFRESH_MAXDELAY
@@ -834,8 +896,6 @@ RetryMiddleware
Failed pages are collected on the scraping process and rescheduled at the
end, once the spider has finished crawling all regular (non failed) pages.
-Once there are no more failed pages to retry, this middleware sends a signal
-(retry_complete), so other extensions could connect to that signal.
The :class:`RetryMiddleware` can be configured through the following
settings (see the settings documentation for more info):
@@ -882,7 +942,7 @@ precedence over the :setting:`RETRY_TIMES` setting.
RETRY_HTTP_CODES
^^^^^^^^^^^^^^^^
-Default: ``[500, 502, 503, 504, 408]``
+Default: ``[500, 502, 503, 504, 522, 524, 408, 429]``
Which HTTP response codes to retry. Other errors (DNS lookup issues,
connections lost, etc) are always retried.
@@ -908,6 +968,24 @@ RobotsTxtMiddleware
To make sure Scrapy respects robots.txt make sure the middleware is enabled
and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
+ The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
+ user agent string to use for matching in the robots.txt_ file. If it
+ is ``None``, the User-Agent header you are sending with the request or the
+ :setting:`USER_AGENT` setting (in that order) will be used for determining
+ the user agent to use in the robots.txt_ file.
+
+ This middleware has to be combined with a robots.txt_ parser.
+
+ Scrapy ships with support for the following robots.txt_ parsers:
+
+ * :ref:`Protego ` (default)
+ * :ref:`RobotFileParser `
+ * :ref:`Reppy `
+ * :ref:`Robotexclusionrulesparser `
+
+ You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+ setting. Or you can also :ref:`implement support for a new parser `.
+
.. reqmeta:: dont_obey_robotstxt
If :attr:`Request.meta ` has
@@ -915,6 +993,129 @@ If :attr:`Request.meta ` has
the request will be ignored by this middleware even if
:setting:`ROBOTSTXT_OBEY` is enabled.
+Parsers vary in several aspects:
+
+* Language of implementation
+
+* Supported specification
+
+* Support for wildcard matching
+
+* Usage of `length based rule `_:
+ in particular for ``Allow`` and ``Disallow`` directives, where the most
+ specific rule based on the length of the path trumps the less specific
+ (shorter) rule
+
+Performance comparison of different parsers is available at `the following link
+`_:
+
+* implemented in Python
+
+* is compliant with `Google's Robots.txt Specification
+ `_
+
+* supports wildcard matching
+
+* uses the length based rule
+
+Scrapy uses this parser by default.
+
+.. _python-robotfileparser:
+
+RobotFileParser
+~~~~~~~~~~~~~~~
+
+Based on `RobotFileParser
+`_:
+
+* is Python's built-in robots.txt_ parser
+
+* is compliant with `Martijn Koster's 1996 draft specification
+ `_
+
+* lacks support for wildcard matching
+
+* doesn't use the length based rule
+
+It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
+
+In order to use this parser, set:
+
+* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
+
+.. _reppy-parser:
+
+Reppy parser
+~~~~~~~~~~~~
+
+Based on `Reppy `_
+
+* is compliant with `Martijn Koster's 1996 draft specification
+ `_
+
+* supports wildcard matching
+
+* uses the length based rule
+
+Native implementation, provides better speed than Protego.
+
+In order to use this parser:
+
+* Install `Reppy `_
+
+* supports wildcard matching
+
+* doesn't use the length based rule
+
+In order to use this parser:
+
+* Install `Robotexclusionrulesparser `.
+
+When this happens, the recommended approach is to
+:ref:`find the data source ` and extract the data
+from it.
+
+If you fail to do that, and you can nonetheless access the desired data through
+the :ref:`DOM ` from your web browser, see
+:ref:`topics-javascript-rendering`.
+
+.. _topics-finding-data-source:
+
+Finding the data source
+=======================
+
+To extract the desired data, you must first find its source location.
+
+If the data is in a non-text-based format, such as an image or a PDF document,
+use the :ref:`network tool ` of your web browser to find
+the corresponding request, and :ref:`reproduce it
+`.
+
+If your web browser lets you select the desired data as text, the data may be
+defined in embedded JavaScript code, or loaded from an external resource in a
+text-based format.
+
+In that case, you can use a tool like wgrep_ to find the URL of that resource.
+
+If the data turns out to come from the original URL itself, you must
+:ref:`inspect the source code of the webpage ` to
+determine where the data is located.
+
+If the data comes from a different URL, you will need to :ref:`reproduce the
+corresponding request `.
+
+.. _topics-inspecting-source:
+
+Inspecting the source code of a webpage
+=======================================
+
+Sometimes you need to inspect the source code of a webpage (not the
+:ref:`DOM `) to determine where some desired data is located.
+
+Use Scrapy’s :command:`fetch` command to download the webpage contents as seen
+by Scrapy::
+
+ scrapy fetch --nolog https://example.com > response.html
+
+If the desired data is in embedded JavaScript code within a ````
+element, see :ref:`topics-parsing-javascript`.
+
+If you cannot find the desired data, first make sure it’s not just Scrapy:
+download the webpage with an HTTP client like curl_ or wget_ and see if the
+information can be found in the response they get.
+
+If they get a response with the desired data, modify your Scrapy
+:class:`~scrapy.http.Request` to match that of the other HTTP client. For
+example, try using the same user-agent string (:setting:`USER_AGENT`) or the
+same :attr:`~scrapy.http.Request.headers`.
+
+If they also get a response without the desired data, you’ll need to take
+steps to make your request more similar to that of the web browser. See
+:ref:`topics-reproducing-requests`.
+
+.. _topics-reproducing-requests:
+
+Reproducing requests
+====================
+
+Sometimes we need to reproduce a request the way our web browser performs it.
+
+Use the :ref:`network tool ` of your web browser to see
+how your web browser performs the desired request, and try to reproduce that
+request with Scrapy.
+
+It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
+method and URL. However, you may also need to reproduce the body, headers and
+form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
+
+As all major browsers allow to export the requests in `cURL
+` inside the network
+tool section.
+
+Once you get the expected response, you can :ref:`extract the desired data from
+it `.
+
+You can reproduce any request with Scrapy. However, some times reproducing all
+necessary requests may not seem efficient in developer time. If that is your
+case, and crawling speed is not a major concern for you, you can alternatively
+consider :ref:`JavaScript pre-rendering `.
+
+If you get the expected response `sometimes`, but not always, the issue is
+probably not your request, but the target server. The target server might be
+buggy, overloaded, or :ref:`banning ` some of your requests.
+
+.. _topics-handling-response-formats:
+
+Handling different response formats
+===================================
+
+Once you have a response with the desired data, how you extract the desired
+data from it depends on the type of response:
+
+- If the response is HTML or XML, use :ref:`selectors
+ ` as usual.
+
+- If the response is JSON, use `json.loads`_ to load the desired data from
+ :attr:`response.text `::
+
+ data = json.loads(response.text)
+
+ If the desired data is inside HTML or XML code embedded within JSON data,
+ you can load that HTML or XML code into a
+ :class:`~scrapy.selector.Selector` and then
+ :ref:`use it ` as usual::
+
+ selector = Selector(data['html'])
+
+- If the response is JavaScript, or HTML with a ```` element
+ containing the desired data, see :ref:`topics-parsing-javascript`.
+
+- If the response is CSS, use a `regular expression`_ to extract the desired
+ data from :attr:`response.text `.
+
+.. _topics-parsing-images:
+
+- If the response is an image or another format based on images (e.g. PDF),
+ read the response as bytes from
+ :attr:`response.body ` and use an OCR
+ solution to extract the desired data as text.
+
+ For example, you can use pytesseract_. To read a table from a PDF,
+ `tabula-py`_ may be a better choice.
+
+- If the response is SVG, or HTML with embedded SVG containing the desired
+ data, you may be able to extract the desired data using
+ :ref:`selectors `, since SVG is based on XML.
+
+ Otherwise, you might need to convert the SVG code into a raster image, and
+ :ref:`handle that raster image `.
+
+.. _topics-parsing-javascript:
+
+Parsing JavaScript code
+=======================
+
+If the desired data is hardcoded in JavaScript, you first need to get the
+JavaScript code:
+
+- If the JavaScript code is in a JavaScript file, simply read
+ :attr:`response.text `.
+
+- If the JavaScript code is within a ```` element of an HTML page,
+ use :ref:`selectors ` to extract the text within that
+ ```` element.
+
+Once you have a string with the JavaScript code, you can extract the desired
+data from it:
+
+- You might be able to use a `regular expression`_ to extract the desired
+ data in JSON format, which you can then parse with `json.loads`_.
+
+ For example, if the JavaScript code contains a separate line like
+ ``var data = {"field": "value"};`` you can extract that data as follows:
+
+ >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
+ >>> json_data = response.css('script::text').re_first(pattern)
+ >>> json.loads(json_data)
+ {'field': 'value'}
+
+- Otherwise, use js2xml_ to convert the JavaScript code into an XML document
+ that you can parse using :ref:`selectors `.
+
+ For example, if the JavaScript code contains
+ ``var data = {field: "value"};`` you can extract that data as follows:
+
+ >>> import js2xml
+ >>> import lxml.etree
+ >>> from parsel import Selector
+ >>> javascript = response.css('script::text').get()
+ >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
+ >>> selector = Selector(text=xml)
+ >>> selector.css('var[name="data"]').get()
+ 'value ` of a webpage, so that
+you can parse it with :ref:`selectors `. It provides great
+flexibility through configuration_ or scripting_.
+
+If you need something beyond what Splash offers, such as interacting with the
+DOM on-the-fly from Python code instead of using a previously-written script,
+or handling multiple web browser windows, you might need to
+:ref:`use a headless browser ` instead.
+
+.. _configuration: https://splash.readthedocs.io/en/stable/api.html
+.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html
+
+.. _topics-headless-browsing:
+
+Using a headless browser
+========================
+
+A `headless browser`_ is a special web browser that provides an API for
+automation.
+
+The easiest way to use a headless browser with Scrapy is to use Selenium_,
+along with `scrapy-selenium`_ for seamless integration.
+
+
+.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
+.. _curl: https://curl.haxx.se/
+.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
+.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
+.. _js2xml: https://github.com/scrapinghub/js2xml
+.. _json.loads: https://docs.python.org/library/json.html#json.loads
+.. _pytesseract: https://github.com/madmaze/pytesseract
+.. _regular expression: https://docs.python.org/library/re.html
+.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+.. _Selenium: https://www.seleniumhq.org/
+.. _Splash: https://github.com/scrapinghub/splash
+.. _tabula-py: https://github.com/chezou/tabula-py
+.. _wget: https://www.gnu.org/software/wget/
+.. _wgrep: https://github.com/stav/wgrep
\ No newline at end of file
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 949cdc638..72bf52227 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -9,19 +9,19 @@ Sending e-mail
Although Python makes sending e-mails relatively easy via the `smtplib`_
library, Scrapy provides its own facility for sending e-mails which is very
-easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
-interfering with the non-blocking IO of the crawler. It also provides a
-simple API for sending attachments and it's very easy to configure, with a few
-:ref:`settings `.
+easy to use and it's implemented using :doc:`Twisted non-blocking IO
+`, to avoid interfering with the non-blocking
+IO of the crawler. It also provides a simple API for sending attachments and
+it's very easy to configure, with a few :ref:`settings
+`.
.. _smtplib: https://docs.python.org/2/library/smtplib.html
-.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
Quick example
=============
There are two ways to instantiate the mail sender. You can instantiate it using
-the standard constructor::
+the standard ``__init__`` method::
from scrapy.mail import MailSender
mailer = MailSender()
@@ -39,7 +39,8 @@ MailSender class reference
==========================
MailSender is the preferred class to use for sending emails from Scrapy, as it
-uses `Twisted non-blocking IO`_, like the rest of the framework.
+uses :doc:`Twisted non-blocking IO `, like the
+rest of the framework.
.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
@@ -111,7 +112,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
Mail settings
=============
-These settings define the default constructor values of the :class:`MailSender`
+These settings define the default ``__init__`` method values of the :class:`MailSender`
class, and can be used to configure e-mail notifications in your project without
writing any code (for those extensions and code that uses :class:`MailSender`).
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 95f7920f8..b8d898022 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -51,7 +51,6 @@ value of one of their fields::
def close_spider(self, spider):
for exporter in self.year_to_exporter.values():
exporter.finish_exporting()
- exporter.file.close()
def _exporter_for_item(self, item):
year = item['year']
@@ -88,8 +87,8 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
-If you use :class:`~.Item` you can declare a serializer in the
-:ref:`field metadata `. The serializer must be
+If you use :class:`~.Item` you can declare a serializer in the
+:ref:`field metadata `. The serializer must be
a callable which receives a value and returns its serialized form.
Example::
@@ -145,7 +144,7 @@ BaseItemExporter
defining what fields to export, whether to export empty fields, or which
encoding to use.
- These features can be configured through the constructor arguments which
+ These features can be configured through the ``__init__`` method arguments which
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
@@ -165,9 +164,9 @@ BaseItemExporter
value unchanged except for ``unicode`` values which are encoded to
``str`` using the encoding declared in the :attr:`encoding` attribute.
- :param field: the field being serialized. If a raw dict is being
+ :param field: the field being serialized. If a raw dict is being
exported (not :class:`~.Item`) *field* value is an empty dict.
- :type field: :class:`~scrapy.item.Field` object or an empty dict
+ :type field: :class:`~scrapy.item.Field` object or an empty dict
:param name: the name of the field being serialized
:type name: str
@@ -223,6 +222,12 @@ BaseItemExporter
* ``indent<=0`` each item on its own line, no indentation
* ``indent>0`` each item on its own line, indented with the provided numeric value
+PythonItemExporter
+------------------
+
+.. autoclass:: PythonItemExporter
+
+
.. highlight:: none
XmlItemExporter
@@ -241,8 +246,8 @@ XmlItemExporter
:param item_element: The name of each item element in the exported XML.
:type item_element: str
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@@ -301,9 +306,9 @@ CsvItemExporter
multi-valued fields, if found.
:type include_headers_line: str
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor, and the leftover arguments to the
- `csv.writer`_ constructor, so you can use any `csv.writer` constructor
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
+ `csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
argument to customize this exporter.
A typical output of this exporter would be::
@@ -329,8 +334,8 @@ PickleItemExporter
For more information, refer to the `pickle module documentation`_.
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
Pickle isn't a human readable format, so no output examples are provided.
@@ -346,8 +351,8 @@ PprintItemExporter
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@@ -362,10 +367,10 @@ JsonItemExporter
.. class:: JsonItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing all
- objects as a list of objects. The additional constructor arguments are
- passed to the :class:`BaseItemExporter` constructor, and the leftover
- arguments to the `JSONEncoder`_ constructor, so you can use any
- `JSONEncoder`_ constructor argument to customize this exporter.
+ objects as a list of objects. The additional ``__init__`` method arguments are
+ passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
+ arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
+ `JSONEncoder`_ ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -393,10 +398,10 @@ JsonLinesItemExporter
.. class:: JsonLinesItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing one
- JSON-encoded item per line. The additional constructor arguments are passed
- to the :class:`BaseItemExporter` constructor, and the leftover arguments to
- the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
- constructor argument to customize this exporter.
+ JSON-encoded item per line. The additional ``__init__`` method arguments are passed
+ to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
+ the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
+ ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -410,3 +415,8 @@ JsonLinesItemExporter
this exporter is well suited for serializing large amounts of data.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
+
+MarshalItemExporter
+-------------------
+
+.. autoclass:: MarshalItemExporter
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index c421a5e05..0a7455ec9 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -19,7 +19,7 @@ settings, just like any other Scrapy code.
It is customary for extensions to prefix their settings with their own name, to
avoid collision with existing (and future) extensions. For example, a
hypothetic extension to handle `Google Sitemaps`_ would use settings like
-`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
+``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
.. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
@@ -28,7 +28,7 @@ Loading & activating extensions
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
-code must be performed in the class constructor (``__init__`` method).
+code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
@@ -183,7 +183,7 @@ Telnet console extension
.. module:: scrapy.extensions.telnet
:synopsis: Telnet console
-.. class:: scrapy.extensions.telnet.TelnetConsole
+.. class:: TelnetConsole
Provides a telnet console for getting into a Python interpreter inside the
currently running Scrapy process, which can be very useful for debugging.
@@ -200,7 +200,7 @@ Memory usage extension
.. module:: scrapy.extensions.memusage
:synopsis: Memory usage extension
-.. class:: scrapy.extensions.memusage.MemoryUsage
+.. class:: MemoryUsage
.. note:: This extension does not work in Windows.
@@ -228,7 +228,7 @@ Memory debugger extension
.. module:: scrapy.extensions.memdebug
:synopsis: Memory debugger extension
-.. class:: scrapy.extensions.memdebug.MemoryDebugger
+.. class:: MemoryDebugger
An extension for debugging memory usage. It collects information about:
@@ -244,7 +244,7 @@ Close spider extension
.. module:: scrapy.extensions.closespider
:synopsis: Close spider extension
-.. class:: scrapy.extensions.closespider.CloseSpider
+.. class:: CloseSpider
Closes a spider automatically when some conditions are met, using a specific
closing reason for each condition.
@@ -317,7 +317,7 @@ StatsMailer extension
.. module:: scrapy.extensions.statsmailer
:synopsis: StatsMailer extension
-.. class:: scrapy.extensions.statsmailer.StatsMailer
+.. class:: StatsMailer
This simple extension can be used to send a notification e-mail every time a
domain has finished scraping, including the Scrapy stats collected. The email
@@ -333,7 +333,7 @@ Debugging extensions
Stack trace dump extension
~~~~~~~~~~~~~~~~~~~~~~~~~~
-.. class:: scrapy.extensions.debug.StackTraceDump
+.. class:: StackTraceDump
Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_
signal is received. The information dumped is the following:
@@ -362,13 +362,13 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
Debugger extension
~~~~~~~~~~~~~~~~~~
-.. class:: scrapy.extensions.debug.Debugger
+.. class:: Debugger
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
signal is received. After the debugger is exited, the Scrapy process continues
running normally.
-For more info see `Debugging in Python`.
+For more info see `Debugging in Python`_.
This extension only works on POSIX-compliant platforms (ie. not Windows).
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 135d05c93..7481b1a99 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -99,12 +99,12 @@ The storages backends supported out of the box are:
* :ref:`topics-feed-storage-fs`
* :ref:`topics-feed-storage-ftp`
- * :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
+ * :ref:`topics-feed-storage-s3` (requires botocore_)
* :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
-or boto_ library is installed (Scrapy supports boto_ only on Python 2).
+library is installed.
.. _topics-feed-uri-params:
@@ -164,6 +164,11 @@ The feeds are stored in a FTP server.
* Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
* Required external libraries: none
+FTP supports two different connection modes: `active or passive
+`_. Scrapy uses the passive connection
+mode by default. To use the active connection mode instead, set the
+:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
+
.. _topics-feed-storage-s3:
S3
@@ -177,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
* ``s3://mybucket/path/to/export.csv``
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- * Required external libraries: `botocore`_ or `boto`_
+ * Required external libraries: `botocore`_
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@@ -185,6 +190,10 @@ passed through the following settings:
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
+You can also define a custom ACL for exported feeds using this setting:
+
+ * :setting:`FEED_STORAGE_S3_ACL`
+
.. _topics-feed-storage-stdout:
Standard output
@@ -205,6 +214,8 @@ These are the settings used for configuring the feed exports:
* :setting:`FEED_URI` (mandatory)
* :setting:`FEED_FORMAT`
* :setting:`FEED_STORAGES`
+ * :setting:`FEED_STORAGE_FTP_ACTIVE`
+ * :setting:`FEED_STORAGE_S3_ACL`
* :setting:`FEED_EXPORTERS`
* :setting:`FEED_STORE_EMPTY`
* :setting:`FEED_EXPORT_ENCODING`
@@ -302,6 +313,30 @@ Default: ``{}``
A dict containing additional feed storage backends supported by your project.
The keys are URI schemes and the values are paths to storage classes.
+.. setting:: FEED_STORAGE_FTP_ACTIVE
+
+FEED_STORAGE_FTP_ACTIVE
+-----------------------
+
+Default: ``False``
+
+Whether to use the active connection mode when exporting feeds to an FTP server
+(``True``) or use the passive connection mode instead (``False``, default).
+
+For information about FTP connection modes, see `What is the difference between
+active and passive FTP? `_.
+
+.. setting:: FEED_STORAGE_S3_ACL
+
+FEED_STORAGE_S3_ACL
+-------------------
+
+Default: ``''`` (empty string)
+
+A string containing a custom ACL for feeds exported to Amazon S3 by your project.
+
+For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs.
+
.. setting:: FEED_STORAGES_BASE
FEED_STORAGES_BASE
@@ -364,5 +399,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
-.. _boto: https://github.com/boto/boto
.. _botocore: https://github.com/boto/botocore
+.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst
deleted file mode 100644
index 4ea8d3bd0..000000000
--- a/docs/topics/firebug.rst
+++ /dev/null
@@ -1,167 +0,0 @@
-.. _topics-firebug:
-
-==========================
-Using Firebug for scraping
-==========================
-
-.. note:: Google Directory, the example website used in this guide is no longer
- available as it `has been shut down by Google`_. The concepts in this guide
- are still valid though. If you want to update this guide to use a new
- (working) site, your contribution will be more than welcome!. See :ref:`topics-contributing`
- for information on how to do so.
-
-Introduction
-============
-
-This document explains how to use `Firebug`_ (a Firefox add-on) to make the
-scraping process easier and more fun. For other useful Firefox add-ons see
-:ref:`topics-firefox-addons`. There are some caveats with using Firefox add-ons
-to inspect pages, see :ref:`topics-firefox-livedom`.
-
-In this example, we'll show how to use `Firebug`_ to scrape data from the
-`Google Directory`_, which contains the same data as the `Open Directory
-Project`_ used in the :ref:`tutorial ` but with a different
-face.
-
-.. _Firebug: https://getfirebug.com/
-.. _Google Directory: http://directory.google.com/
-.. _Open Directory Project: http://www.dmoz.org
-
-Firebug comes with a very useful feature called `Inspect Element`_ which allows
-you to inspect the HTML code of the different page elements just by hovering
-your mouse over them. Otherwise you would have to search for the tags manually
-through the HTML body which can be a very tedious task.
-
-.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
-
-In the following screenshot you can see the `Inspect Element`_ tool in action.
-
-.. image:: _images/firebug1.png
- :width: 913
- :height: 600
- :alt: Inspecting elements with Firebug
-
-At first sight, we can see that the directory is divided in categories, which
-are also divided in subcategories.
-
-However, it seems that there are more subcategories than the ones being shown
-in this page, so we'll keep looking:
-
-.. image:: _images/firebug2.png
- :width: 819
- :height: 629
- :alt: Inspecting elements with Firebug
-
-As expected, the subcategories contain links to other subcategories, and also
-links to actual websites, which is the purpose of the directory.
-
-Getting links to follow
-=======================
-
-By looking at the category URLs we can see they share a pattern:
-
- http://directory.google.com/Category/Subcategory/Another_Subcategory
-
-Once we know that, we are able to construct a regular expression to follow
-those links. For example, the following one::
-
- directory\.google\.com/[A-Z][a-zA-Z_/]+$
-
-So, based on that regular expression we can create the first crawling rule::
-
- Rule(LinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$', ),
- 'parse_category',
- follow=True,
- ),
-
-The :class:`~scrapy.spiders.Rule` object instructs
-:class:`~scrapy.spiders.CrawlSpider` based spiders how to follow the
-category links. ``parse_category`` will be a method of the spider which will
-process and extract data from those pages.
-
-This is how the spider would look so far::
-
- from scrapy.linkextractors import LinkExtractor
- from scrapy.spiders import CrawlSpider, Rule
-
- class GoogleDirectorySpider(CrawlSpider):
- name = 'directory.google.com'
- allowed_domains = ['directory.google.com']
- start_urls = ['http://directory.google.com/']
-
- rules = (
- Rule(LinkExtractor(allow='directory\.google\.com/[A-Z][a-zA-Z_/]+$'),
- 'parse_category', follow=True,
- ),
- )
-
- def parse_category(self, response):
- # write the category page data extraction code here
- pass
-
-
-Extracting the data
-===================
-
-Now we're going to write the code to extract data from those pages.
-
-With the help of Firebug, we'll take a look at some page containing links to
-websites (say http://directory.google.com/Top/Arts/Awards/) and find out how we can
-extract those links using :ref:`Selectors `. We'll also
-use the :ref:`Scrapy shell ` to test those XPath's and make sure
-they work as we expect.
-
-.. image:: _images/firebug3.png
- :width: 965
- :height: 751
- :alt: Inspecting elements with Firebug
-
-As you can see, the page markup is not very descriptive: the elements don't
-contain ``id``, ``class`` or any attribute that clearly identifies them, so
-we'll use the ranking bars as a reference point to select the data to extract
-when we construct our XPaths.
-
-After using FireBug, we can see that each link is inside a ``td`` tag, which is
-itself inside a ``tr`` tag that also contains the link's ranking bar (in
-another ``td``).
-
-So we can select the ranking bar, then find its parent (the ``tr``), and then
-finally, the link's ``td`` (which contains the data we want to scrape).
-
-This results in the following XPath::
-
- //td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td//a
-
-It's important to use the :ref:`Scrapy shell ` to test these
-complex XPath expressions and make sure they work as expected.
-
-Basically, that expression will look for the ranking bar's ``td`` element, and
-then select any ``td`` element who has a descendant ``a`` element whose
-``href`` attribute contains the string ``#pagerank``"
-
-Of course, this is not the only XPath, and maybe not the simpler one to select
-that data. Another approach could be, for example, to find any ``font`` tags
-that have that grey colour of the links,
-
-Finally, we can write our ``parse_category()`` method::
-
- def parse_category(self, response):
- # The path to website links in directory page
- links = response.xpath('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font')
-
- for link in links:
- item = DirectoryItem()
- item['name'] = link.xpath('a/text()').extract()
- item['url'] = link.xpath('a/@href').extract()
- item['description'] = link.xpath('font[2]/text()').extract()
- yield item
-
-
-Be aware that you may find some elements which appear in Firebug but
-not in the original HTML, such as the typical case of ````
-elements.
-
-or tags which Therefer in page HTML
-sources may on Firebug inspects the live DOM
-
-.. _has been shut down by Google: https://searchenginewatch.com/sew/news/2096661/google-directory-shut
diff --git a/docs/topics/firefox.rst b/docs/topics/firefox.rst
deleted file mode 100644
index 2c85848be..000000000
--- a/docs/topics/firefox.rst
+++ /dev/null
@@ -1,82 +0,0 @@
-.. _topics-firefox:
-
-==========================
-Using Firefox for scraping
-==========================
-
-Here is a list of tips and advice on using Firefox for scraping, along with a
-list of useful Firefox add-ons to ease the scraping process.
-
-.. _topics-firefox-livedom:
-
-Caveats with inspecting the live browser DOM
-============================================
-
-Since Firefox add-ons operate on a live browser DOM, what you'll actually see
-when inspecting the page source is not the original HTML, but a modified one
-after applying some browser clean up and executing Javascript code. Firefox,
-in particular, is known for adding `` `` elements to tables. Scrapy, on
-the other hand, does not modify the original page HTML, so you won't be able to
-extract any data if you use `` `` in your XPath expressions.
-
-Therefore, you should keep in mind the following things when working with
-Firefox and XPath:
-
-* Disable Firefox Javascript while inspecting the DOM looking for XPaths to be
- used in Scrapy
-
-* Never use full XPath paths, use relative and clever ones based on attributes
- (such as ``id``, ``class``, ``width``, etc) or any identifying features like
- ``contains(@href, 'image')``.
-
-* Never include `` `` elements in your XPath expressions unless you
- really know what you're doing
-
-.. _topics-firefox-addons:
-
-Useful Firefox add-ons for scraping
-===================================
-
-Firebug
--------
-
-`Firebug`_ is a widely known tool among web developers and it's also very
-useful for scraping. In particular, its `Inspect Element`_ feature comes very
-handy when you need to construct the XPaths for extracting data because it
-allows you to view the HTML code of each page element while moving your mouse
-over it.
-
-See :ref:`topics-firebug` for a detailed guide on how to use Firebug with
-Scrapy.
-
-XPather
--------
-
-`XPather`_ allows you to test XPath expressions directly on the pages.
-
-XPath Checker
--------------
-
-`XPath Checker`_ is another Firefox add-on for testing XPaths on your pages.
-
-Tamper Data
------------
-
-`Tamper Data`_ is a Firefox add-on which allows you to view and modify the HTTP
-request headers sent by Firefox. Firebug also allows to view HTTP headers, but
-not to modify them.
-
-Firecookie
-----------
-
-`Firecookie`_ makes it easier to view and manage cookies. You can use this
-extension to create a new cookie, delete existing cookies, see a list of cookies
-for the current site, manage cookies permissions and a lot more.
-
-.. _Firebug: https://getfirebug.com/
-.. _Inspect Element: https://www.youtube.com/watch?v=-pT_pDe54aA
-.. _XPather: https://addons.mozilla.org/en-US/firefox/addon/xpather/
-.. _XPath Checker: https://addons.mozilla.org/en-US/firefox/addon/xpath-checker/
-.. _Tamper Data: https://addons.mozilla.org/en-US/firefox/addon/tamper-data/
-.. _Firecookie: https://addons.mozilla.org/en-US/firefox/addon/firecookie/
-
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 38265b474..cdc4953c2 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
This method is called for every item pipeline component. :meth:`process_item`
must either: return a dict with data, return an :class:`~scrapy.item.Item`
- (or any descendant class) object, return a `Twisted Deferred`_ or raise
+ (or any descendant class) object, return a
+ :class:`~twisted.internet.defer.Deferred` or raise
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
processed by further pipeline components.
@@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
:type crawler: :class:`~scrapy.crawler.Crawler` object
-.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
-
Item pipeline example
=====================
@@ -87,8 +86,8 @@ contain a price::
vat_factor = 1.15
def process_item(self, item, spider):
- if item['price']:
- if item['price_excludes_vat']:
+ if item.get('price'):
+ if item.get('price_excludes_vat'):
item['price'] = item['price'] * self.vat_factor
return item
else:
@@ -166,7 +165,8 @@ method and how to clean up the resources properly.::
Take screenshot of item
-----------------------
-This example demonstrates how to return Deferred_ from :meth:`process_item` method.
+This example demonstrates how to return a
+:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
It uses Splash_ to render screenshot of item url. Pipeline
makes request to locally running instance of Splash_. After request is downloaded
and Deferred callback fires, it saves item to a file and adds filename to an item.
@@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
return item
.. _Splash: https://splash.readthedocs.io/en/stable/
-.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
Duplicates filter
-----------------
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 4423bbda2..15313775b 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -16,12 +16,12 @@ especially in a larger project with many spiders.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide a `dictionary-like`_ API with a convenient syntax for declaring
-their available fields.
+their available fields.
-Various Scrapy components use extra information provided by Items:
+Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help find memory leaks
+tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
@@ -40,6 +40,7 @@ objects. Here is an example::
name = scrapy.Field()
price = scrapy.Field()
stock = scrapy.Field()
+ tags = scrapy.Field()
last_updated = scrapy.Field(serializer=str)
.. note:: Those familiar with `Django`_ will notice that Scrapy Items are
@@ -83,105 +84,128 @@ notice the API is very similar to the `dict API`_.
Creating items
--------------
-::
+>>> product = Product(name='Desktop PC', price=1000)
+>>> print(product)
+Product(name='Desktop PC', price=1000)
- >>> product = Product(name='Desktop PC', price=1000)
- >>> print product
- Product(name='Desktop PC', price=1000)
Getting field values
--------------------
-::
+>>> product['name']
+Desktop PC
+>>> product.get('name')
+Desktop PC
- >>> product['name']
- Desktop PC
- >>> product.get('name')
- Desktop PC
+>>> product['price']
+1000
- >>> product['price']
- 1000
+>>> product['last_updated']
+Traceback (most recent call last):
+ ...
+KeyError: 'last_updated'
- >>> product['last_updated']
- Traceback (most recent call last):
- ...
- KeyError: 'last_updated'
+>>> product.get('last_updated', 'not set')
+not set
- >>> product.get('last_updated', 'not set')
- not set
+>>> product['lala'] # getting unknown field
+Traceback (most recent call last):
+ ...
+KeyError: 'lala'
- >>> product['lala'] # getting unknown field
- Traceback (most recent call last):
- ...
- KeyError: 'lala'
+>>> product.get('lala', 'unknown field')
+'unknown field'
- >>> product.get('lala', 'unknown field')
- 'unknown field'
+>>> 'name' in product # is name field populated?
+True
- >>> 'name' in product # is name field populated?
- True
+>>> 'last_updated' in product # is last_updated populated?
+False
- >>> 'last_updated' in product # is last_updated populated?
- False
+>>> 'last_updated' in product.fields # is last_updated a declared field?
+True
- >>> 'last_updated' in product.fields # is last_updated a declared field?
- True
+>>> 'lala' in product.fields # is lala a declared field?
+False
- >>> 'lala' in product.fields # is lala a declared field?
- False
Setting field values
--------------------
-::
+>>> product['last_updated'] = 'today'
+>>> product['last_updated']
+today
- >>> product['last_updated'] = 'today'
- >>> product['last_updated']
- today
+>>> product['lala'] = 'test' # setting unknown field
+Traceback (most recent call last):
+ ...
+KeyError: 'Product does not support field: lala'
- >>> product['lala'] = 'test' # setting unknown field
- Traceback (most recent call last):
- ...
- KeyError: 'Product does not support field: lala'
Accessing all populated values
------------------------------
-To access all populated values, just use the typical `dict API`_::
+To access all populated values, just use the typical `dict API`_:
- >>> product.keys()
- ['price', 'name']
+>>> product.keys()
+['price', 'name']
+
+>>> product.items()
+[('price', 1000), ('name', 'Desktop PC')]
+
+
+.. _copying-items:
+
+Copying items
+-------------
+
+To copy an item, you must first decide whether you want a shallow copy or a
+deep copy.
+
+If your item contains mutable_ values like lists or dictionaries, a shallow
+copy will keep references to the same mutable values across all different
+copies.
+
+.. _mutable: https://docs.python.org/glossary.html#term-mutable
+
+For example, if you have an item with a list of tags, and you create a shallow
+copy of that item, both the original item and the copy have the same list of
+tags. Adding a tag to the list of one of the items will add the tag to the
+other item as well.
+
+If that is not the desired behavior, use a deep copy instead.
+
+See the `documentation of the copy module`_ for more information.
+
+.. _documentation of the copy module: https://docs.python.org/library/copy.html
+
+To create a shallow copy of an item, you can either call
+:meth:`~scrapy.item.Item.copy` on an existing item
+(``product2 = product.copy()``) or instantiate your item class from an existing
+item (``product2 = Product(product)``).
+
+To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
+(``product2 = product.deepcopy()``).
- >>> product.items()
- [('price', 1000), ('name', 'Desktop PC')]
Other common tasks
------------------
-Copying items::
+Creating dicts from items:
- >>> product2 = Product(product)
- >>> print product2
- Product(name='Desktop PC', price=1000)
+>>> dict(product) # create a dict from all populated values
+{'price': 1000, 'name': 'Desktop PC'}
- >>> product3 = product2.copy()
- >>> print product3
- Product(name='Desktop PC', price=1000)
+Creating items from dicts:
-Creating dicts from items::
+>>> Product({'name': 'Laptop PC', 'price': 1500})
+Product(price=1500, name='Laptop PC')
- >>> dict(product) # create a dict from all populated values
- {'price': 1000, 'name': 'Desktop PC'}
+>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
+Traceback (most recent call last):
+ ...
+KeyError: 'Product does not support field: lala'
-Creating items from dicts::
-
- >>> Product({'name': 'Laptop PC', 'price': 1500})
- Product(price=1500, name='Laptop PC')
-
- >>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
- Traceback (most recent call last):
- ...
- KeyError: 'Product does not support field: lala'
Extending Items
===============
@@ -211,8 +235,12 @@ Item objects
Return a new Item optionally initialized from the given argument.
- Items replicate the standard `dict API`_, including its constructor. The
- only additional attribute provided by Items is:
+ Items replicate the standard `dict API`_, including its ``__init__`` method, and
+ also provide the following additional API members:
+
+ .. automethod:: copy
+
+ .. automethod:: deepcopy
.. attribute:: fields
@@ -237,3 +265,9 @@ Field objects
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
+Other classes related to Item
+=============================
+
+.. autoclass:: BaseItem
+
+.. autoclass:: ItemMeta
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 06c7fff3d..8816a028c 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -30,7 +30,7 @@ a *single* job.
How to use it
=============
-To start a spider with persistence supported enabled, run it like this::
+To start a spider with persistence support enabled, run it like this::
scrapy crawl somespider -s JOBDIR=crawls/somespider-1
@@ -71,33 +71,11 @@ on cookies.
Request serialization
---------------------
-Requests must be serializable by the `pickle` module, in order for persistence
-to work, so you should make sure that your requests are serializable.
-
-The most common issue here is to use ``lambda`` functions on request callbacks that
-can't be persisted.
-
-So, for example, this won't work::
-
- def some_callback(self, response):
- somearg = 'test'
- return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
-
- def other_callback(self, response, somearg):
- print "the argument passed is:", somearg
-
-But this will::
-
- def some_callback(self, response):
- somearg = 'test'
- return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
-
- def other_callback(self, response):
- somearg = response.meta['somearg']
- print "the argument passed is:", somearg
+For persistence to work, :class:`~scrapy.http.Request` objects must be
+serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
+values passed to their ``__init__`` method, which must be methods of the
+running :class:`~scrapy.spiders.Spider` class.
If you wish to log the requests that couldn't be serialized, you can set the
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
It is ``False`` by default.
-
-.. _pickle: https://docs.python.org/library/pickle.html
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 92590c180..9fee333ac 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -27,10 +27,11 @@ Common causes of memory leaks
It happens quite often (sometimes by accident, sometimes on purpose) that the
Scrapy developer passes objects referenced in Requests (for example, using the
-:attr:`~scrapy.http.Request.meta` attribute or the request callback function)
-and that effectively bounds the lifetime of those referenced objects to the
-lifetime of the Request. This is, by far, the most common cause of memory leaks
-in Scrapy projects, and a quite difficult one to debug for newcomers.
+:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
+attributes or the request callback function) and that effectively bounds the
+lifetime of those referenced objects to the lifetime of the Request. This is,
+by far, the most common cause of memory leaks in Scrapy projects, and a quite
+difficult one to debug for newcomers.
In big projects, the spiders are typically written by different people and some
of those spiders could be "leaking" and thus affecting the rest of the other
@@ -48,7 +49,8 @@ Too Many Requests?
By default Scrapy keeps the request queue in memory; it includes
:class:`~scrapy.http.Request` objects and all objects
-referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
+referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
+and :attr:`~scrapy.http.Request.meta`).
While not necessarily a leak, this can take a lot of memory. Enabling
:ref:`persistent job queue ` could help keeping memory usage
in control.
@@ -101,14 +103,14 @@ Let's see a concrete example of a hypothetical case of memory leaks.
Suppose we have some spider with a line similar to this one::
return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
- callback=self.parse, meta={referer: response})
+ callback=self.parse, cb_kwargs={'referer': response})
That line is passing a response reference inside a request which effectively
ties the response lifetime to the requests' one, and that would definitely
cause memory leaks.
Let's see how we can discover the cause (without knowing it
-a-priori, of course) by using the ``trackref`` tool.
+a priori, of course) by using the ``trackref`` tool.
After the crawler is running for a few minutes and we notice its memory usage
has grown a lot, we can enter its telnet console and check the live
@@ -130,21 +132,21 @@ and check the code of the spider to discover the nasty line that is
generating the leaks (passing response references inside requests).
Sometimes extra information about live objects can be helpful.
-Let's check the oldest response::
+Let's check the oldest response:
- >>> from scrapy.utils.trackref import get_oldest
- >>> r = get_oldest('HtmlResponse')
- >>> r.url
- 'http://www.somenastyspider.com/product.php?pid=123'
+>>> from scrapy.utils.trackref import get_oldest
+>>> r = get_oldest('HtmlResponse')
+>>> r.url
+'http://www.somenastyspider.com/product.php?pid=123'
If you want to iterate over all objects, instead of getting the oldest one, you
-can use the :func:`scrapy.utils.trackref.iter_all` function::
+can use the :func:`scrapy.utils.trackref.iter_all` function:
- >>> from scrapy.utils.trackref import iter_all
- >>> [r.url for r in iter_all('HtmlResponse')]
- ['http://www.somenastyspider.com/product.php?pid=123',
- 'http://www.somenastyspider.com/product.php?pid=584',
- ...
+>>> from scrapy.utils.trackref import iter_all
+>>> [r.url for r in iter_all('HtmlResponse')]
+['http://www.somenastyspider.com/product.php?pid=123',
+ 'http://www.somenastyspider.com/product.php?pid=584',
+...]
Too many spiders?
-----------------
@@ -153,10 +155,10 @@ If your project has too many spiders executed in parallel,
the output of :func:`prefs()` can be difficult to read.
For this reason, that function has a ``ignore`` argument which can be used to
ignore a particular class (and all its subclases). For
-example, this won't show any live references to spiders::
+example, this won't show any live references to spiders:
- >>> from scrapy.spiders import Spider
- >>> prefs(ignore=Spider)
+>>> from scrapy.spiders import Spider
+>>> prefs(ignore=Spider)
.. module:: scrapy.utils.trackref
:synopsis: Track references of live objects
@@ -202,6 +204,7 @@ memory leaks (Requests, Responses, Items, and Selectors). However, there are
other cases where the memory leaks could come from other (more or less obscure)
objects. If this is your case, and you can't find your leaks using ``trackref``,
you still have another resource: the `Guppy library`_.
+If you're using Python3, see :ref:`topics-leaks-muppy`.
.. _Guppy library: https://pypi.python.org/pypi/guppy
@@ -211,41 +214,41 @@ If you use ``pip``, you can install Guppy with the following command::
The telnet console also comes with a built-in shortcut (``hpy``) for accessing
Guppy heap objects. Here's an example to view all Python objects available in
-the heap using Guppy::
+the heap using Guppy:
- >>> x = hpy.heap()
- >>> x.bytype
- Partition of a set of 297033 objects. Total size = 52587824 bytes.
- Index Count % Size % Cumulative % Type
- 0 22307 8 16423880 31 16423880 31 dict
- 1 122285 41 12441544 24 28865424 55 str
- 2 68346 23 5966696 11 34832120 66 tuple
- 3 227 0 5836528 11 40668648 77 unicode
- 4 2461 1 2222272 4 42890920 82 type
- 5 16870 6 2024400 4 44915320 85 function
- 6 13949 5 1673880 3 46589200 89 types.CodeType
- 7 13422 5 1653104 3 48242304 92 list
- 8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
- 9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
- <1676 more rows. Type e.g. '_.more' to view.>
+>>> x = hpy.heap()
+>>> x.bytype
+Partition of a set of 297033 objects. Total size = 52587824 bytes.
+ Index Count % Size % Cumulative % Type
+ 0 22307 8 16423880 31 16423880 31 dict
+ 1 122285 41 12441544 24 28865424 55 str
+ 2 68346 23 5966696 11 34832120 66 tuple
+ 3 227 0 5836528 11 40668648 77 unicode
+ 4 2461 1 2222272 4 42890920 82 type
+ 5 16870 6 2024400 4 44915320 85 function
+ 6 13949 5 1673880 3 46589200 89 types.CodeType
+ 7 13422 5 1653104 3 48242304 92 list
+ 8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern
+ 9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers
+<1676 more rows. Type e.g. '_.more' to view.>
You can see that most space is used by dicts. Then, if you want to see from
-which attribute those dicts are referenced, you could do::
+which attribute those dicts are referenced, you could do:
- >>> x.bytype[0].byvia
- Partition of a set of 22307 objects. Total size = 16423880 bytes.
- Index Count % Size % Cumulative % Referred Via:
- 0 10982 49 9416336 57 9416336 57 '.__dict__'
- 1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
- 2 3097 14 1122904 7 13220744 80
- 3 990 4 277200 2 13497944 82 "['cookies']"
- 4 987 4 276360 2 13774304 84 "['cache']"
- 5 985 4 275800 2 14050104 86 "['meta']"
- 6 897 4 251160 2 14301264 87 '[2]'
- 7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
- 8 672 3 188160 1 14686312 89 "['cb_kwargs']"
- 9 27 0 155016 1 14841328 90 '[1]'
- <333 more rows. Type e.g. '_.more' to view.>
+>>> x.bytype[0].byvia
+Partition of a set of 22307 objects. Total size = 16423880 bytes.
+ Index Count % Size % Cumulative % Referred Via:
+ 0 10982 49 9416336 57 9416336 57 '.__dict__'
+ 1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals'
+ 2 3097 14 1122904 7 13220744 80
+ 3 990 4 277200 2 13497944 82 "['cookies']"
+ 4 987 4 276360 2 13774304 84 "['cache']"
+ 5 985 4 275800 2 14050104 86 "['meta']"
+ 6 897 4 251160 2 14301264 87 '[2]'
+ 7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']"
+ 8 672 3 188160 1 14686312 89 "['cb_kwargs']"
+ 9 27 0 155016 1 14841328 90 '[1]'
+<333 more rows. Type e.g. '_.more' to view.>
As you can see, the Guppy module is very powerful but also requires some deep
knowledge about Python internals. For more info about Guppy, refer to the
@@ -253,6 +256,50 @@ knowledge about Python internals. For more info about Guppy, refer to the
.. _Guppy documentation: http://guppy-pe.sourceforge.net/
+.. _topics-leaks-muppy:
+
+Debugging memory leaks with muppy
+=================================
+You can use muppy from `Pympler`_.
+
+.. _Pympler: https://pypi.org/project/Pympler/
+
+If you use ``pip``, you can install muppy with the following command::
+
+ pip install Pympler
+
+Here's an example to view all Python objects available in
+the heap using muppy:
+
+>>> from pympler import muppy
+>>> all_objects = muppy.get_objects()
+>>> len(all_objects)
+28667
+>>> from pympler import summary
+>>> suml = summary.summarize(all_objects)
+>>> summary.print_(suml)
+ types | # objects | total size
+==================================== | =========== | ============
+ ` returns a
+list of matching :class:`scrapy.link.Link` objects from a
+:class:`~scrapy.http.Response` object.
+Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
+through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
+extractors in regular spiders.
.. _topics-link-extractors-ref:
-Built-in link extractors reference
-==================================
+Link extractor reference
+========================
.. module:: scrapy.linkextractors
:synopsis: Link extractors classes
-Link extractors classes bundled with Scrapy are provided in the
-:mod:`scrapy.linkextractors` module.
-
-The default link extractor is ``LinkExtractor``, which is the same as
-:class:`~.LxmlLinkExtractor`::
+The link extractor class is
+:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
+can also be imported as ``scrapy.linkextractors.LinkExtractor``::
from scrapy.linkextractors import LinkExtractor
-There used to be other link extractor classes in previous Scrapy versions,
-but they are deprecated now.
-
LxmlLinkExtractor
-----------------
@@ -93,6 +80,12 @@ LxmlLinkExtractor
Has the same behaviour as ``restrict_xpaths``.
:type restrict_css: str or list
+ :param restrict_text: a single regular expression (or list of regular expressions)
+ that the link's text must match in order to be extracted. If not
+ given (or empty), it will match all links. If a list of regular expressions is
+ given, the link will be extracted if it matches at least one.
+ :type restrict_text: a regular expression (or list of)
+
:param tags: a tag or a list of tags to consider when extracting links.
Defaults to ``('a', 'area')``.
:type tags: str or list
@@ -146,4 +139,6 @@ LxmlLinkExtractor
from elements or attributes which allow leading/trailing whitespaces).
:type strip: boolean
+ .. automethod:: extract_links
+
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index a895b535c..9d5fccbbc 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -26,7 +26,7 @@ Using Item Loaders to populate items
To use an Item Loader, you must first instantiate it. You can either
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
-which case an Item is automatically instantiated in the Item Loader constructor
+which case an Item is automatically instantiated in the Item Loader ``__init__`` method
using the Item class specified in the :attr:`ItemLoader.default_item_class`
attribute.
@@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
the same item field; the Item Loader will know how to "join" those values later
using a proper processing function.
+.. note:: Collected data is internally stored as lists,
+ allowing to add several values to the same field.
+ If an ``item`` argument is passed when creating a loader,
+ each of the item's values will be stored as-is if it's already
+ an iterable, or wrapped with a list if it's a single value.
+
Here is a typical Item Loader usage in a :ref:`Spider `, using
the :ref:`Product item ` declared in the :ref:`Items
chapter `::
@@ -128,28 +134,14 @@ So what happens is:
It's worth noticing that processors are just callable objects, which are called
with the data to be parsed, and return a parsed value. So you can use any
function as input or output processor. The only requirement is that they must
-accept one (and only one) positional argument, which will be an iterator.
+accept one (and only one) positional argument, which will be an iterable.
-.. note:: Both input and output processors must receive an iterator as their
+.. note:: Both input and output processors must receive an iterable as their
first argument. The output of those functions can be anything. The result of
input processors will be appended to an internal list (in the Loader)
containing the collected values (for that field). The result of the output
processors is the value that will be finally assigned to the item.
-If you want to use a plain function as a processor, make sure it receives
-``self`` as the first argument::
-
- def lowercase_processor(self, values):
- for v in values:
- yield v.lower()
-
- class MyItemLoader(ItemLoader):
- name_in = lowercase_processor
-
-This is because whenever a function is assigned as a class variable, it becomes
-a method and would be passed the instance as the the first argument when being
-called. See `this answer on stackoverflow`_ for more details.
-
The other thing you need to keep in mind is that the values returned by input
processors are collected internally (in lists) and then passed to output
processors to populate the fields.
@@ -157,7 +149,7 @@ processors to populate the fields.
Last, but not least, Scrapy comes with some :ref:`commonly used processors
` built-in for convenience.
-.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
+
Declaring Item Loaders
======================
@@ -214,14 +206,12 @@ metadata. Here is an example::
output_processor=TakeFirst(),
)
-::
-
- >>> from scrapy.loader import ItemLoader
- >>> il = ItemLoader(item=Product())
- >>> il.add_value('name', [u'Welcome to my', u'website '])
- >>> il.add_value('price', [u'€', u'1000 '])
- >>> il.load_item()
- {'name': u'Welcome to my website', 'price': u'1000'}
+>>> from scrapy.loader import ItemLoader
+>>> il = ItemLoader(item=Product())
+>>> il.add_value('name', [u'Welcome to my', u'website '])
+>>> il.add_value('price', [u'€', u'1000 '])
+>>> il.load_item()
+{'name': u'Welcome to my website', 'price': u'1000'}
The precedence order, for both input and output processors, is as follows:
@@ -265,7 +255,7 @@ There are several ways to modify Item Loader context values:
loader.context['unit'] = 'cm'
2. On Item Loader instantiation (the keyword arguments of Item Loader
- constructor are stored in the Item Loader context)::
+ ``__init__`` method are stored in the Item Loader context)::
loader = ItemLoader(product, unit='cm')
@@ -286,7 +276,7 @@ ItemLoader objects
given, one is instantiated automatically using the class in
:attr:`default_item_class`.
- When instantiated with a `selector` or a `response` parameters
+ When instantiated with a ``selector`` or a ``response`` parameters
the :class:`ItemLoader` class provides convenient mechanisms for extracting
data from web pages using :ref:`selectors `.
@@ -322,11 +312,11 @@ ItemLoader objects
applied before processors
:type re: str or compiled regex
- Examples::
+ Examples:
- >>> from scrapy.loader.processors import TakeFirst
- >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
- 'FOO`
+ >>> from scrapy.loader.processors import TakeFirst
+ >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
+ 'FOO`
.. method:: add_value(field_name, value, \*processors, \**kwargs)
@@ -485,6 +475,8 @@ ItemLoader objects
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
+ This is mostly used as a property so when attempting to override this
+ value, you may want to check out :attr:`default_item_class` first.
.. attribute:: context
@@ -494,7 +486,7 @@ ItemLoader objects
.. attribute:: default_item_class
An Item class (or factory), used to instantiate items when not given in
- the constructor.
+ the ``__init__`` method.
.. attribute:: default_input_processor
@@ -509,15 +501,15 @@ ItemLoader objects
.. attribute:: default_selector_class
The class used to construct the :attr:`selector` of this
- :class:`ItemLoader`, if only a response is given in the constructor.
- If a selector is given in the constructor this attribute is ignored.
+ :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
+ If a selector is given in the ``__init__`` method this attribute is ignored.
This attribute is sometimes overridden in subclasses.
.. attribute:: selector
The :class:`~scrapy.selector.Selector` object to extract data from.
- It's either the selector given in the constructor or one created from
- the response given in the constructor using the
+ It's either the selector given in the ``__init__`` method or one created from
+ the response given in the ``__init__`` method using the
:attr:`default_selector_class`. This attribute is meant to be
read-only.
@@ -642,46 +634,46 @@ Here is a list of all built-in processors:
.. class:: Identity
The simplest processor, which doesn't do anything. It returns the original
- values unchanged. It doesn't receive any constructor arguments, nor does it
+ values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
accept Loader contexts.
- Example::
+ Example:
- >>> from scrapy.loader.processors import Identity
- >>> proc = Identity()
- >>> proc(['one', 'two', 'three'])
- ['one', 'two', 'three']
+ >>> from scrapy.loader.processors import Identity
+ >>> proc = Identity()
+ >>> proc(['one', 'two', 'three'])
+ ['one', 'two', 'three']
.. class:: TakeFirst
Returns the first non-null/non-empty value from the values received,
so it's typically used as an output processor to single-valued fields.
- It doesn't receive any constructor arguments, nor does it accept Loader contexts.
+ It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
- Example::
+ Example:
- >>> from scrapy.loader.processors import TakeFirst
- >>> proc = TakeFirst()
- >>> proc(['', 'one', 'two', 'three'])
- 'one'
+ >>> from scrapy.loader.processors import TakeFirst
+ >>> proc = TakeFirst()
+ >>> proc(['', 'one', 'two', 'three'])
+ 'one'
.. class:: Join(separator=u' ')
- Returns the values joined with the separator given in the constructor, which
+ Returns the values joined with the separator given in the ``__init__`` method, which
defaults to ``u' '``. It doesn't accept Loader contexts.
When using the default separator, this processor is equivalent to the
function: ``u' '.join``
- Examples::
+ Examples:
- >>> from scrapy.loader.processors import Join
- >>> proc = Join()
- >>> proc(['one', 'two', 'three'])
- u'one two three'
- >>> proc = Join('` through that parameter.
- The keyword arguments passed in the constructor are used as the default
+ The keyword arguments passed in the ``__init__`` method are used as the default
Loader context values passed to each function call. However, the final
Loader context values passed to functions are overridden with the currently
active Loader context accessible through the :meth:`ItemLoader.context`
@@ -738,41 +730,41 @@ Here is a list of all built-in processors:
:meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
`, which returns a list of unicode strings.
- The example below should clarify how it works::
+ The example below should clarify how it works:
- >>> def filter_world(x):
- ... return None if x == 'world' else x
- ...
- >>> from scrapy.loader.processors import MapCompose
- >>> proc = MapCompose(filter_world, unicode.upper)
- >>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
- [u'HELLO, u'THIS', u'IS', u'SCRAPY']
+ >>> def filter_world(x):
+ ... return None if x == 'world' else x
+ ...
+ >>> from scrapy.loader.processors import MapCompose
+ >>> proc = MapCompose(filter_world, str.upper)
+ >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
+ ['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
- constructor keyword arguments are used as default context values. See
+ ``__init__`` method keyword arguments are used as default context values. See
:class:`Compose` processor for more info.
.. class:: SelectJmes(json_path)
- Queries the value using the json path provided to the constructor and returns the output.
+ Queries the value using the json path provided to the ``__init__`` method and returns the output.
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
This processor takes only one input at a time.
- Example::
+ Example:
- >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
- >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
- >>> proc({'foo': 'bar'})
- 'bar'
- >>> proc({'foo': {'bar': 'baz'}})
- {'bar': 'baz'}
+ >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
+ >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
+ >>> proc({'foo': 'bar'})
+ 'bar'
+ >>> proc({'foo': {'bar': 'baz'}})
+ {'bar': 'baz'}
- Working with Json::
+ Working with Json:
- >>> import json
- >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
- >>> proc_single_json_str('{"foo": "bar"}')
- u'bar'
- >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
- >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
- [u'bar']
+ >>> import json
+ >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
+ >>> proc_single_json_str('{"foo": "bar"}')
+ 'bar'
+ >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
+ >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
+ ['bar']
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 0986929ad..d4d22d889 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -171,9 +171,9 @@ listed in `logging's logrecord attributes docs
`_
respectively.
-If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy
+If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
component that prints the log. It is unset by default, hence logs contain the
-scrapy component responsible for that log output.
+Scrapy component responsible for that log output.
Command-line options
--------------------
@@ -193,6 +193,18 @@ to override some of the Scrapy settings regarding logging.
Module `logging.handlers `_
Further documentation on available handlers
+.. _custom-log-formats:
+
+Custom Log Formats
+------------------
+
+A custom log format can be set for different actions by extending
+:class:`~scrapy.logformatter.LogFormatter` class and making
+:setting:`LOG_FORMATTER` point to your new class.
+
+.. autoclass:: scrapy.logformatter.LogFormatter
+ :members:
+
Advanced customization
----------------------
@@ -238,22 +250,23 @@ scrapy.utils.log module
.. autofunction:: configure_logging
- ``configure_logging`` is automatically called when using Scrapy commands,
- but needs to be called explicitly when running custom scripts. In that
- case, its usage is not required but it's recommended.
+ ``configure_logging`` is automatically called when using Scrapy commands
+ or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly
+ when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
+ In that case, its usage is not required but it's recommended.
- If you plan on configuring the handlers yourself is still recommended you
- call this function, passing `install_root_handler=False`. Bear in mind
- there won't be any log output set by default in that case.
+ Another option when running custom scripts is to manually configure the logging.
+ To do this you can use `logging.basicConfig()`_ to set a basic root handler.
- To get you started on manually configuring logging's output, you can use
- `logging.basicConfig()`_ to set a basic root handler. This is an example
- on how to redirect ``INFO`` or higher messages to a file::
+ Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
+ so it is recommended to only use `logging.basicConfig()`_ together with
+ :class:`~scrapy.crawler.CrawlerRunner`.
+
+ This is an example on how to redirect ``INFO`` or higher messages to a file::
import logging
from scrapy.utils.log import configure_logging
- configure_logging(install_root_handler=False)
logging.basicConfig(
filename='log.txt',
format='%(levelname)s: %(message)s',
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index a1f518cbd..1e0e0f18f 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -97,7 +97,6 @@ For Files Pipeline, use::
ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
-
.. note::
You can also use both the Files and Images Pipeline at the same time.
@@ -132,7 +131,7 @@ For example, the following image URL::
http://www.example.com/image.jpg
-Whose `SHA1 hash` is::
+Whose ``SHA1 hash`` is::
3afec3b4765f8f0a07b78f98c07b83f013567a0a
@@ -148,6 +147,25 @@ Where:
* ``full`` is a sub-directory to separate full images from thumbnails (if
used). For more info see :ref:`topics-images-thumbnails`.
+FTP server storage
+------------------
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server.
+Scrapy will automatically upload the files to the server.
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` should be written in one of the
+following forms::
+
+ ftp://username:password@address:port/path
+ ftp://address:port/path
+
+If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
+:setting:`FTP_PASSWORD` settings respectively.
+
+FTP supports two different connection modes: active or passive. Scrapy uses
+the passive connection mode by default. To use the active connection mode instead,
+set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
+
Amazon S3 storage
-----------------
@@ -171,7 +189,7 @@ policy::
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
-Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
+Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
@@ -310,7 +328,7 @@ images.
.. setting:: IMAGES_THUMBS
-In order use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
+In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
where the keys are the thumbnail names and the values are their dimensions.
For example::
@@ -392,6 +410,36 @@ See here the methods that you can override in your custom Files Pipeline:
.. class:: FilesPipeline
+ .. method:: file_path(request, response, info)
+
+ This method is called once per downloaded item. It returns the
+ download path of the file originating from the specified
+ :class:`response `.
+
+ In addition to ``response``, this method receives the original
+ :class:`request ` and
+ :class:`info `.
+
+ You can override this method to customize the download path of each file.
+
+ For example, if file URLs end like regular paths (e.g.
+ ``https://example.com/a/b/c/foo.png``), you can use the following
+ approach to download all files into the ``files`` folder with their
+ original filenames (e.g. ``files/foo.png``)::
+
+ import os
+ from urllib.parse import urlparse
+
+ from scrapy.pipelines.files import FilesPipeline
+
+ class MyFilesPipeline(FilesPipeline):
+
+ def file_path(self, request, response, info):
+ return 'files/' + os.path.basename(urlparse(request.url).path)
+
+ By default the :meth:`file_path` method returns
+ ``full/.``.
+
.. method:: FilesPipeline.get_media_requests(item, info)
As seen on the workflow, the pipeline will get the URLs of the images to
@@ -411,8 +459,9 @@ See here the methods that you can override in your custom Files Pipeline:
* ``success`` is a boolean which is ``True`` if the image was downloaded
successfully or ``False`` if it failed for some reason
- * ``file_info_or_error`` is a dict containing the following keys (if success
- is ``True``) or a `Twisted Failure`_ if there was a problem.
+ * ``file_info_or_error`` is a dict containing the following keys (if
+ success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
+ there was a problem.
* ``url`` - the url where the file was downloaded from. This is the url of
the request returned from the :meth:`~get_media_requests`
@@ -475,6 +524,36 @@ See here the methods that you can override in your custom Images Pipeline:
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
customizing the field names and adding custom behavior for images.
+ .. method:: file_path(request, response, info)
+
+ This method is called once per downloaded item. It returns the
+ download path of the file originating from the specified
+ :class:`response `.
+
+ In addition to ``response``, this method receives the original
+ :class:`request ` and
+ :class:`info `.
+
+ You can override this method to customize the download path of each file.
+
+ For example, if file URLs end like regular paths (e.g.
+ ``https://example.com/a/b/c/foo.png``), you can use the following
+ approach to download all files into the ``files`` folder with their
+ original filenames (e.g. ``files/foo.png``)::
+
+ import os
+ from urllib.parse import urlparse
+
+ from scrapy.pipelines.images import ImagesPipeline
+
+ class MyImagesPipeline(ImagesPipeline):
+
+ def file_path(self, request, response, info):
+ return 'files/' + os.path.basename(urlparse(request.url).path)
+
+ By default the :meth:`file_path` method returns
+ ``full/.``.
+
.. method:: ImagesPipeline.get_media_requests(item, info)
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
@@ -497,7 +576,7 @@ See here the methods that you can override in your custom Images Pipeline:
Custom Images pipeline example
==============================
-Here is a full example of the Images Pipeline whose methods are examplified
+Here is a full example of the Images Pipeline whose methods are exemplified
above::
import scrapy
@@ -517,5 +596,12 @@ above::
item['image_paths'] = image_paths
return item
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
+
+To enable your custom media pipeline component you must add its class import path to the
+:setting:`ITEM_PIPELINES` setting, like in the following example::
+
+ ITEM_PIPELINES = {
+ 'myproject.pipelines.MyImagesPipeline': 300
+ }
+
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 02cfa9b05..e3e8fdc72 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -34,15 +34,16 @@ Here's an example showing how to run a single spider with it.
# Your spider definition
...
- process = CrawlerProcess({
- 'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+ process = CrawlerProcess(settings={
+ 'FEED_FORMAT': 'json',
+ 'FEED_URI': 'items.json'
})
process.crawl(MySpider)
process.start() # the script will block here until the crawling is finished
-Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
-acquainted with its usage details.
+Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
+documentation to get acquainted with its usage details.
If you are inside a Scrapy project there are some additional helpers you can
use to import those components within the project. You can automatically import
@@ -80,7 +81,7 @@ returned by the :meth:`CrawlerRunner.crawl
` method.
Here's an example of its usage, along with a callback to manually stop the
-reactor after `MySpider` has finished running.
+reactor after ``MySpider`` has finished running.
::
@@ -100,7 +101,7 @@ reactor after `MySpider` has finished running.
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
-.. seealso:: `Twisted Reactor Overview`_.
+.. seealso:: :doc:`twisted:core/howto/reactor-basics`
.. _run-multiple-spiders:
@@ -252,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _testspiders: https://github.com/scrapinghub/testspiders
-.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index e29914dbf..8997a7f19 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
Request objects
===============
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
+.. autoclass:: Request
A :class:`Request` object represents an HTTP request, which is usually
generated in the Spider and executed by the Downloader, and thus generating
@@ -50,7 +50,7 @@ Request objects
:type meta: dict
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
- ``str`` using the `encoding` passed (which defaults to ``utf-8``). If
+ ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
``body`` is not given, an empty string is stored. Regardless of the
type of this argument, the final value stored will be a ``str`` (never
``unicode`` or ``None``).
@@ -83,17 +83,21 @@ Request objects
.. reqmeta:: dont_merge_cookies
When some site returns cookies (in a response) those are stored in the
- cookies for that domain and will be sent again in future requests. That's
- the typical behaviour of any regular web browser. However, if, for some
- reason, you want to avoid merging with existing cookies you can instruct
- Scrapy to do so by setting the ``dont_merge_cookies`` key to True in the
- :attr:`Request.meta`.
+ cookies for that domain and will be sent again in future requests.
+ That's the typical behaviour of any regular web browser.
- Example of request without merging cookies::
+ To create a request that does not send stored cookies and does not
+ store received cookies, set the ``dont_merge_cookies`` key to ``True``
+ in :attr:`request.meta `.
- request_with_cookies = Request(url="http://www.example.com",
- cookies={'currency': 'USD', 'country': 'UY'},
- meta={'dont_merge_cookies': True})
+ Example of a request that sends manually-defined cookies and ignores
+ cookie storage::
+
+ Request(
+ url="http://www.example.com",
+ cookies={'currency': 'USD', 'country': 'UY'},
+ meta={'dont_merge_cookies': True},
+ )
For more info see :ref:`cookies-mw`.
:type cookies: dict or list
@@ -117,8 +121,8 @@ Request objects
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
- with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
- as first parameter.
+ with 404 HTTP errors and such. It receives a
+ :exc:`~twisted.python.failure.Failure` as first parameter.
For more information,
see :ref:`topics-request-response-ref-errbacks` below.
:type errback: callable
@@ -126,11 +130,14 @@ Request objects
:param flags: Flags sent to the request, can be used for logging or similar purposes.
:type flags: list
+ :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+ :type cb_kwargs: dict
+
.. attribute:: Request.url
A string containing the URL of this request. Keep in mind that this
attribute contains the escaped URL, so it can differ from the URL passed in
- the constructor.
+ the ``__init__`` method.
This attribute is read-only. To change the URL of a Request use
:meth:`replace`.
@@ -165,6 +172,17 @@ Request objects
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.meta`` attribute.
+ .. attribute:: Request.cb_kwargs
+
+ A dictionary that contains arbitrary metadata for this request. Its contents
+ will be passed to the Request's callback as keyword arguments. It is empty
+ for new Requests, which means by default callbacks only get a :class:`Response`
+ object as argument.
+
+ This dict is `shallow copied`_ when the request is cloned using the
+ ``copy()`` or ``replace()`` methods, and can also be accessed, in your
+ spider, from the ``response.cb_kwargs`` attribute.
+
.. _shallow copied: https://docs.python.org/2/library/copy.html
.. method:: Request.copy()
@@ -172,14 +190,16 @@ Request objects
Return a new Request which is a copy of this Request. See also:
:ref:`topics-request-response-ref-request-callback-arguments`.
- .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback])
+ .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
Return a Request object with the same members, except for those members
given new values by whichever keyword arguments are specified. The
- attribute :attr:`Request.meta` is copied by default (unless a new value
- is given in the ``meta`` argument). See also
+ :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow
+ copied by default (unless new values are given as arguments). See also
:ref:`topics-request-response-ref-request-callback-arguments`.
+ .. automethod:: from_curl
+
.. _topics-request-response-ref-request-callback-arguments:
Passing additional data to callback functions
@@ -200,25 +220,31 @@ Example::
self.logger.info("Visited %s", response.url)
In some cases you may be interested in passing arguments to those callback
-functions so you can receive the arguments later, in the second callback. You
-can use the :attr:`Request.meta` attribute for that.
+functions so you can receive the arguments later, in the second callback.
+The following example shows how to achieve this by using the
+:attr:`Request.cb_kwargs` attribute:
-Here's an example of how to pass an item using this mechanism, to populate
-different fields from different pages::
+::
- def parse_page1(self, response):
- item = MyItem()
- item['main_url'] = response.url
- request = scrapy.Request("http://www.example.com/some_page.html",
- callback=self.parse_page2)
- request.meta['item'] = item
+ def parse(self, response):
+ request = scrapy.Request('http://www.example.com/index.html',
+ callback=self.parse_page2,
+ cb_kwargs=dict(main_url=response.url))
+ request.cb_kwargs['foo'] = 'bar' # add more arguments for the callback
yield request
- def parse_page2(self, response):
- item = response.meta['item']
- item['other_url'] = response.url
- yield item
+ def parse_page2(self, response, main_url, foo):
+ yield dict(
+ main_url=main_url,
+ other_url=response.url,
+ foo=foo,
+ )
+.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
+ Prior to that, using :attr:`Request.meta` was recommended for passing
+ information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs`
+ became the preferred way for handling user information, leaving :attr:`Request.meta`
+ for communication with components like middlewares and extensions.
.. _topics-request-response-ref-errbacks:
@@ -228,8 +254,8 @@ Using errbacks to catch exceptions in request processing
The errback of a request is a function that will be called when an exception
is raise while processing it.
-It receives a `Twisted Failure`_ instance as first parameter and can be
-used to track connection establishment timeouts, DNS errors etc.
+It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
+be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed::
@@ -299,6 +325,7 @@ Those are:
* :reqmeta:`dont_merge_cookies`
* :reqmeta:`cookiejar`
* :reqmeta:`dont_cache`
+* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`bindaddress`
* :reqmeta:`dont_obey_robotstxt`
@@ -373,7 +400,7 @@ fields with form data from :class:`Response` objects.
.. class:: FormRequest(url, [formdata, ...])
- The :class:`FormRequest` class adds a new argument to the constructor. The
+ The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
@@ -446,7 +473,7 @@ fields with form data from :class:`Response` objects.
:type dont_click: boolean
The other parameters of this class method are passed directly to the
- :class:`FormRequest` constructor.
+ :class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
@@ -489,6 +516,11 @@ method for this job. Here's an example spider which uses it::
import scrapy
+ def authentication_failed(response):
+ # TODO: Check the contents of the response and return True if it failed
+ # or False if it succeeded.
+ pass
+
class LoginSpider(scrapy.Spider):
name = 'example.com'
start_urls = ['http://www.example.com/users/login.php']
@@ -501,18 +533,55 @@ method for this job. Here's an example spider which uses it::
)
def after_login(self, response):
- # check login succeed before going on
- if "authentication failed" in response.body:
+ if authentication_failed(response):
self.logger.error("Login failed")
return
# continue scraping with authenticated session...
+JsonRequest
+-----------
+
+The JsonRequest class extends the base :class:`Request` class with functionality for
+dealing with JSON requests.
+
+.. class:: JsonRequest(url, [... data, dumps_kwargs])
+
+ The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
+ remaining arguments are the same as for the :class:`Request` class and are
+ not documented here.
+
+ Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json``
+ and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01``
+
+ :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
+ if :attr:`Request.body` argument is provided this parameter will be ignored.
+ if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
+ set to ``'POST'`` automatically.
+ :type data: JSON serializable object
+
+ :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
+ data into JSON format.
+ :type dumps_kwargs: dict
+
+.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
+
+JsonRequest usage example
+-------------------------
+
+Sending a JSON POST request with a JSON payload::
+
+ data = {
+ 'name1': 'value1',
+ 'name2': 'value2',
+ }
+ yield JsonRequest(url='http://www.example.com/post/action', data=data)
+
Response objects
================
-.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
+.. autoclass:: Response
A :class:`Response` object represents an HTTP response, which is usually
downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -527,8 +596,8 @@ Response objects
(for single valued headers) or lists (for multi-valued headers).
:type headers: dict
- :param body: the response body. To access the decoded text as str (unicode
- in Python 2) you can use ``response.text`` from an encoding-aware
+ :param body: the response body. To access the decoded text as str you can use
+ ``response.text`` from an encoding-aware
:ref:`Response subclass `,
such as :class:`TextResponse`.
:type body: bytes
@@ -606,7 +675,7 @@ Response objects
.. attribute:: Response.flags
A list that contains flags for this response. Flags are labels used for
- tagging Responses. For example: `'cached'`, `'redirected`', etc. And
+ tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
they're shown on the string representation of the Response (`__str__`
method) which is used by the engine for logging.
@@ -632,6 +701,8 @@ Response objects
.. automethod:: Response.follow
+ .. automethod:: Response.follow_all
+
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
@@ -652,7 +723,7 @@ TextResponse objects
:class:`Response` class, which is meant to be used only for binary data,
such as images, sounds or any media file.
- :class:`TextResponse` objects support a new constructor argument, in
+ :class:`TextResponse` objects support a new ``__init__`` method argument, in
addition to the base :class:`Response` objects. The remaining functionality
is the same as for the :class:`Response` class and is not documented here.
@@ -678,7 +749,7 @@ TextResponse objects
``unicode(response.body)`` is not a correct way to convert response
body to unicode: you would be using the system default encoding
- (typically `ascii`) instead of the response encoding.
+ (typically ``ascii``) instead of the response encoding.
.. attribute:: TextResponse.encoding
@@ -686,7 +757,7 @@ TextResponse objects
A string with the encoding of this response. The encoding is resolved by
trying the following mechanisms, in order:
- 1. the encoding passed in the constructor `encoding` argument
+ 1. the encoding passed in the ``__init__`` method ``encoding`` argument
2. the encoding declared in the Content-Type HTTP header. If this
encoding is not valid (ie. unknown), it is ignored and the next
@@ -721,10 +792,12 @@ TextResponse objects
.. automethod:: TextResponse.follow
+ .. automethod:: TextResponse.follow_all
+
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is
- kept for backwards compatibility; please prefer ``response.text``.
+ kept for backward compatibility; please prefer ``response.text``.
HtmlResponse objects
@@ -747,5 +820,4 @@ XmlResponse objects
adds encoding auto-discovering support by looking into the XML declaration
line. See :attr:`TextResponse.encoding`.
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 8ac40c3cc..8ec758b0e 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -6,7 +6,7 @@ Selectors
When you're scraping web pages, the most common task you need to perform is
to extract data from the HTML source. There are several libraries available to
-achieve this:
+achieve this, such as:
* `BeautifulSoup`_ is a very popular web scraping library among Python
programmers which constructs a Python object based on the structure of the
@@ -25,16 +25,14 @@ either by `XPath`_ or `CSS`_ expressions.
used with HTML. `CSS`_ is a language for applying styles to HTML documents. It
defines selectors to associate those styles with specific HTML elements.
-Scrapy selectors are built over the `lxml`_ library, which means they're very
-similar in speed and parsing accuracy.
+.. note::
+ Scrapy Selectors is a thin wrapper around `parsel`_ library; the purpose of
+ this wrapper is to provide better integration with Scrapy Response objects.
-This page explains how selectors work and describes their API which is very
-small and simple, unlike the `lxml`_ API which is much bigger because the
-`lxml`_ library can be used for many other tasks, besides selecting markup
-documents.
-
-For a complete reference of the selectors API see
-:ref:`Selector reference `
+ `parsel`_ is a stand-alone web scraping library which can be used without
+ Scrapy. It uses `lxml`_ library under the hood, and implements an
+ easy API on top of lxml API. It means Scrapy selectors are very similar
+ in speed and parsing accuracy to lxml.
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: http://lxml.de/
@@ -42,7 +40,7 @@ For a complete reference of the selectors API see
.. _cssselect: https://pypi.python.org/pypi/cssselect/
.. _XPath: https://www.w3.org/TR/xpath
.. _CSS: https://www.w3.org/TR/selectors
-
+.. _parsel: https://parsel.readthedocs.io/
Using selectors
===============
@@ -52,45 +50,61 @@ Constructing selectors
.. highlight:: python
+Response objects expose a :class:`~scrapy.selector.Selector` instance
+on ``.selector`` attribute:
+
+>>> response.selector.xpath('//span/text()').get()
+'good'
+
+Querying responses using XPath and CSS is so common that responses include two
+more shortcuts: ``response.xpath()`` and ``response.css()``:
+
+>>> response.xpath('//span/text()').get()
+'good'
+>>> response.css('span::text').get()
+'good'
+
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
-constructed by passing **text** or :class:`~scrapy.http.TextResponse`
-object. It automatically chooses the best parsing rules (XML vs HTML) based on
-input type::
+constructed by passing either :class:`~scrapy.http.TextResponse` object or
+markup as an unicode string (in ``text`` argument).
+Usually there is no need to construct Scrapy selectors manually:
+``response`` object is available in Spider callbacks, so in most cases
+it is more convenient to use ``response.css()`` and ``response.xpath()``
+shortcuts. By using ``response.selector`` or one of these shortcuts
+you can also ensure the response body is parsed only once.
- >>> from scrapy.selector import Selector
- >>> from scrapy.http import HtmlResponse
+But if required, it is possible to use ``Selector`` directly.
+Constructing from text:
-Constructing from text::
+>>> from scrapy.selector import Selector
+>>> body = 'good '
+>>> Selector(text=body).xpath('//span/text()').get()
+'good'
- >>> body = 'good '
- >>> Selector(text=body).xpath('//span/text()').extract()
- [u'good']
+Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
+:class:`~scrapy.http.TextResponse` subclasses:
-Constructing from response::
-
- >>> response = HtmlResponse(url='http://example.com', body=body)
- >>> Selector(response=response).xpath('//span/text()').extract()
- [u'good']
-
-For convenience, response objects expose a selector on `.selector` attribute,
-it's totally OK to use this shortcut when possible::
-
- >>> response.selector.xpath('//span/text()').extract()
- [u'good']
+>>> from scrapy.selector import Selector
+>>> from scrapy.http import HtmlResponse
+>>> response = HtmlResponse(url='http://example.com', body=body)
+>>> Selector(response=response).xpath('//span/text()').get()
+'good'
+``Selector`` automatically chooses the best parsing rules
+(XML vs HTML) based on input type.
Using selectors
---------------
-To explain how to use the selectors we'll use the `Scrapy shell` (which
+To explain how to use the selectors we'll use the ``Scrapy shell`` (which
provides interactive testing) and an example page located in the Scrapy
documentation server:
- https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+ https://docs.scrapy.org/en/latest/_static/selectors-sample1.html
.. _topics-selectors-htmlcode:
-Here's its HTML code:
+For the sake of completeness, here's its full HTML code:
.. literalinclude:: ../_static/selectors-sample1.html
:language: html
@@ -99,7 +113,7 @@ Here's its HTML code:
First, let's open the shell::
- scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+ scrapy shell https://docs.scrapy.org/en/latest/_static/selectors-sample1.html
Then, after the shell loads, you'll have the response available as ``response``
shell variable, and its attached selector in ``response.selector`` attribute.
@@ -109,92 +123,193 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
.. highlight:: python
So, by looking at the :ref:`HTML code ` of that
-page, let's construct an XPath for selecting the text inside the title tag::
+page, let's construct an XPath for selecting the text inside the title tag:
- >>> response.selector.xpath('//title/text()')
- []
+>>> response.xpath('//title/text()')
+[]
-Querying responses using XPath and CSS is so common that responses include two
-convenience shortcuts: ``response.xpath()`` and ``response.css()``::
+To actually extract the textual data, you must call the selector ``.get()``
+or ``.getall()`` methods, as follows:
- >>> response.xpath('//title/text()')
- []
- >>> response.css('title::text')
- []
+>>> response.xpath('//title/text()').getall()
+['Example website']
+>>> response.xpath('//title/text()').get()
+'Example website'
+
+``.get()`` always returns a single result; if there are several matches,
+content of a first match is returned; if there are no matches, None
+is returned. ``.getall()`` returns a list with all results.
+
+Notice that CSS selectors can select text or attribute nodes using CSS3
+pseudo-elements:
+
+>>> response.css('title::text').get()
+'Example website'
As you can see, ``.xpath()`` and ``.css()`` methods return a
:class:`~scrapy.selector.SelectorList` instance, which is a list of new
-selectors. This API can be used for quickly selecting nested data::
+selectors. This API can be used for quickly selecting nested data:
- >>> response.css('img').xpath('@src').extract()
- [u'image1_thumb.jpg',
- u'image2_thumb.jpg',
- u'image3_thumb.jpg',
- u'image4_thumb.jpg',
- u'image5_thumb.jpg']
+>>> response.css('img').xpath('@src').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
-To actually extract the textual data, you must call the selector ``.extract()``
-method, as follows::
+If you want to extract only the first matched element, you can call the
+selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
+previous Scrapy versions):
- >>> response.xpath('//title/text()').extract()
- [u'Example website']
-
-If you want to extract only first matched element, you can call the selector ``.extract_first()``
-
- >>> response.xpath('//div[@id="images"]/a/text()').extract_first()
- u'Name: My image 1 '
+>>> response.xpath('//div[@id="images"]/a/text()').get()
+'Name: My image 1 '
It returns ``None`` if no element was found:
- >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None
- True
+>>> response.xpath('//div[@id="not-exists"]/text()').get() is None
+True
-A default return value can be provided as an argument, to be used instead of ``None``:
+A default return value can be provided as an argument, to be used instead
+of ``None``:
- >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found')
- 'not-found'
+>>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
+'not-found'
-Notice that CSS selectors can select text or attribute nodes using CSS3
-pseudo-elements::
+Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
+using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
- >>> response.css('title::text').extract()
- [u'Example website']
+>>> [img.attrib['src'] for img in response.css('img')]
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
-Now we're going to get the base URL and some image links::
+As a shortcut, ``.attrib`` is also available on SelectorList directly;
+it returns attributes for the first matching element:
- >>> response.xpath('//base/@href').extract()
- [u'http://example.com/']
+>>> response.css('img').attrib['src']
+'image1_thumb.jpg'
- >>> response.css('base::attr(href)').extract()
- [u'http://example.com/']
+This is most useful when only a single result is expected, e.g. when selecting
+by id, or selecting unique elements on a web page:
- >>> response.xpath('//a[contains(@href, "image")]/@href').extract()
- [u'image1.html',
- u'image2.html',
- u'image3.html',
- u'image4.html',
- u'image5.html']
+>>> response.css('base').attrib['href']
+'http://example.com/'
- >>> response.css('a[href*=image]::attr(href)').extract()
- [u'image1.html',
- u'image2.html',
- u'image3.html',
- u'image4.html',
- u'image5.html']
+Now we're going to get the base URL and some image links:
- >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract()
- [u'image1_thumb.jpg',
- u'image2_thumb.jpg',
- u'image3_thumb.jpg',
- u'image4_thumb.jpg',
- u'image5_thumb.jpg']
+>>> response.xpath('//base/@href').get()
+'http://example.com/'
- >>> response.css('a[href*=image] img::attr(src)').extract()
- [u'image1_thumb.jpg',
- u'image2_thumb.jpg',
- u'image3_thumb.jpg',
- u'image4_thumb.jpg',
- u'image5_thumb.jpg']
+>>> response.css('base::attr(href)').get()
+'http://example.com/'
+
+>>> response.css('base').attrib['href']
+'http://example.com/'
+
+>>> response.xpath('//a[contains(@href, "image")]/@href').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
+
+>>> response.css('a[href*=image]::attr(href)').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
+
+>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
+
+>>> response.css('a[href*=image] img::attr(src)').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
+
+.. _topics-selectors-css-extensions:
+
+Extensions to CSS Selectors
+---------------------------
+
+Per W3C standards, `CSS selectors`_ do not support selecting text nodes
+or attribute values.
+But selecting these is so essential in a web scraping context
+that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
+
+* to select text nodes, use ``::text``
+* to select attribute values, use ``::attr(name)`` where *name* is the
+ name of the attribute that you want the value of
+
+.. warning::
+ These pseudo-elements are Scrapy-/Parsel-specific.
+ They will most probably not work with other libraries like
+ `lxml`_ or `PyQuery`_.
+
+.. _PyQuery: https://pypi.python.org/pypi/pyquery
+
+Examples:
+
+* ``title::text`` selects children text nodes of a descendant ```` element:
+
+>>> response.css('title::text').get()
+'Example website'
+
+* ``*::text`` selects all descendant text nodes of the current selector context:
+
+>>> response.css('#images *::text').getall()
+['\n ',
+ 'Name: My image 1 ',
+ '\n ',
+ 'Name: My image 2 ',
+ '\n ',
+ 'Name: My image 3 ',
+ '\n ',
+ 'Name: My image 4 ',
+ '\n ',
+ 'Name: My image 5 ',
+ '\n ']
+
+* ``foo::text`` returns no results if ``foo`` element exists, but contains
+ no text (i.e. text is empty):
+
+>>> response.css('img::text').getall()
+[]
+
+ This means ``.css('foo::text').get()`` could return None even if an element
+ exists. Use ``default=''`` if you always want a string:
+
+>>> response.css('img::text').get()
+>>> response.css('img::text').get(default='')
+''
+
+* ``a::attr(href)`` selects the *href* attribute value of descendant links:
+
+>>> response.css('a::attr(href)').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
+
+.. note::
+ See also: :ref:`selecting-attributes`.
+
+.. note::
+ You cannot chain these pseudo-elements. But in practice it would not
+ make much sense: text nodes do not have attributes, and attribute values
+ are string values already and do not have children nodes.
+
+.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
.. _topics-selectors-nesting-selectors:
@@ -203,25 +318,67 @@ Nesting selectors
The selection methods (``.xpath()`` or ``.css()``) return a list of selectors
of the same type, so you can call the selection methods for those selectors
-too. Here's an example::
+too. Here's an example:
- >>> links = response.xpath('//a[contains(@href, "image")]')
- >>> links.extract()
- [u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
- u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
- u'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
- u'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
- u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
+>>> links = response.xpath('//a[contains(@href, "image")]')
+>>> links.getall()
+['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
+ '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
+ '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
+ '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
+ '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
- >>> for index, link in enumerate(links):
- ... args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract())
- ... print 'Link number %d points to url %s and image %s' % args
+>>> for index, link in enumerate(links):
+... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
+... print('Link number %d points to url %r and image %r' % args)
+Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
+Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
+Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
+Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
+Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
- Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg']
- Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg']
- Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg']
- Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg']
- Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg']
+.. _selecting-attributes:
+
+Selecting element attributes
+----------------------------
+
+There are several ways to get a value of an attribute. First, one can use
+XPath syntax:
+
+>>> response.xpath("//a/@href").getall()
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+XPath syntax has a few advantages: it is a standard XPath feature, and
+``@attributes`` can be used in other parts of an XPath expression - e.g.
+it is possible to filter by attribute value.
+
+Scrapy also provides an extension to CSS selectors (``::attr(...)``)
+which allows to get attribute values:
+
+>>> response.css('a::attr(href)').getall()
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+In addition to that, there is a ``.attrib`` property of Selector.
+You can use it if you prefer to lookup attributes in Python
+code, without using XPaths or CSS extensions:
+
+>>> [a.attrib['href'] for a in response.css('a')]
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+This property is also available on SelectorList; it returns a dictionary
+with attributes of a first matching element. It is convenient to use when
+a selector is expected to give a single result (e.g. when selecting by element
+ID, or when selecting an unique element on a page):
+
+>>> response.css('base').attrib
+{'href': 'http://example.com/'}
+>>> response.css('base').attrib['href']
+'http://example.com/'
+
+``.attrib`` property of an empty SelectorList is empty:
+
+>>> response.css('foo').attrib
+{}
Using selectors with regular expressions
----------------------------------------
@@ -232,20 +389,86 @@ data using regular expressions. However, unlike using ``.xpath()`` or
can't construct nested ``.re()`` calls.
Here's an example used to extract image names from the :ref:`HTML code
-<topics-selectors-htmlcode>` above::
+<topics-selectors-htmlcode>` above:
- >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
- [u'My image 1',
- u'My image 2',
- u'My image 3',
- u'My image 4',
- u'My image 5']
+>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
+['My image 1',
+ 'My image 2',
+ 'My image 3',
+ 'My image 4',
+ 'My image 5']
-There's an additional helper reciprocating ``.extract_first()`` for ``.re()``,
-named ``.re_first()``. Use it to extract just the first matching string::
+There's an additional helper reciprocating ``.get()`` (and its
+alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
+Use it to extract just the first matching string:
+
+>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
+'My image 1'
+
+.. _old-extraction-api:
+
+extract() and extract_first()
+-----------------------------
+
+If you're a long-time Scrapy user, you're probably familiar
+with ``.extract()`` and ``.extract_first()`` selector methods. Many blog posts
+and tutorials are using them as well. These methods are still supported
+by Scrapy, there are **no plans** to deprecate them.
+
+However, Scrapy usage docs are now written using ``.get()`` and
+``.getall()`` methods. We feel that these new methods result in a more concise
+and readable code.
+
+The following examples show how these methods map to each other.
+
+1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:
+
+ >>> response.css('a::attr(href)').get()
+ 'image1.html'
+ >>> response.css('a::attr(href)').extract_first()
+ 'image1.html'
+
+2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:
+
+ >>> response.css('a::attr(href)').getall()
+ ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+ >>> response.css('a::attr(href)').extract()
+ ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+
+3. ``Selector.get()`` is the same as ``Selector.extract()``:
+
+ >>> response.css('a::attr(href)')[0].get()
+ 'image1.html'
+ >>> response.css('a::attr(href)')[0].extract()
+ 'image1.html'
+
+4. For consistency, there is also ``Selector.getall()``, which returns a list:
+
+ >>> response.css('a::attr(href)')[0].getall()
+ ['image1.html']
+
+So, the main difference is that output of ``.get()`` and ``.getall()`` methods
+is more predictable: ``.get()`` always returns a single result, ``.getall()``
+always returns a list of all extracted results. With ``.extract()`` method
+it was not always obvious if a result is a list or not; to get a single
+result either ``.extract()`` or ``.extract_first()`` should be called.
+
+
+.. _topics-selectors-xpaths:
+
+Working with XPaths
+===================
+
+Here are some tips which may help you to use XPath with Scrapy selectors
+effectively. If you are not much familiar with XPath yet,
+you may want to take a look first at this `XPath tutorial`_.
+
+.. note::
+ Some of the tips are based on `this post from ScrapingHub's blog`_.
+
+.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
+.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
- >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
- u'My image 1'
.. _topics-selectors-relative-xpaths:
@@ -257,32 +480,140 @@ with ``/``, that XPath will be absolute to the document and not relative to the
``Selector`` you're calling it from.
For example, suppose you want to extract all ``<p>`` elements inside ``<div>``
-elements. First, you would get all ``<div>`` elements::
+elements. First, you would get all ``<div>`` elements:
- >>> divs = response.xpath('//div')
+>>> divs = response.xpath('//div')
At first, you may be tempted to use the following approach, which is wrong, as
it actually extracts all ``<p>`` elements from the document, not only those
-inside ``<div>`` elements::
+inside ``<div>`` elements:
- >>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document
- ... print p.extract()
+>>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document
+... print(p.get())
-This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
+This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):
- >>> for p in divs.xpath('.//p'): # extracts all <p> inside
- ... print p.extract()
+>>> for p in divs.xpath('.//p'): # extracts all <p> inside
+... print(p.get())
-Another common case would be to extract all direct ``<p>`` children::
+Another common case would be to extract all direct ``<p>`` children:
- >>> for p in divs.xpath('p'):
- ... print p.extract()
+>>> for p in divs.xpath('p'):
+... print(p.get())
For more details about relative XPaths see the `Location Paths`_ section in the
XPath specification.
.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
+When querying by class, consider using CSS
+------------------------------------------
+
+Because an element can contain multiple CSS classes, the XPath way to select elements
+by class is the rather verbose::
+
+ *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
+
+If you use ``@class='someclass'`` you may end up missing elements that have
+other classes, and if you just use ``contains(@class, 'someclass')`` to make up
+for that you may end up with more elements that you want, if they have a different
+class name that shares the string ``someclass``.
+
+As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
+you can just select by class using CSS and then switch to XPath when needed:
+
+>>> from scrapy import Selector
+>>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
+>>> sel.css('.shout').xpath('./time/@datetime').getall()
+['2014-07-23 19:00']
+
+This is cleaner than using the verbose XPath trick shown above. Just remember
+to use the ``.`` in the XPath expressions that will follow.
+
+Beware of the difference between //node[1] and (//node)[1]
+----------------------------------------------------------
+
+``//node[1]`` selects all the nodes occurring first under their respective parents.
+
+``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
+
+Example:
+
+>>> from scrapy import Selector
+>>> sel = Selector(text="""
+....: <ul class="list">
+....: <li>1</li>
+....: <li>2</li>
+....: <li>3</li>
+....: </ul>
+....: <ul class="list">
+....: <li>4</li>
+....: <li>5</li>
+....: <li>6</li>
+....: </ul>""")
+>>> xp = lambda x: sel.xpath(x).getall()
+
+This gets all first ``<li>`` elements under whatever it is its parent:
+
+>>> xp("//li[1]")
+['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>`` element in the whole document:
+
+>>> xp("(//li)[1]")
+['<li>1</li>']
+
+This gets all first ``<li>`` elements under an ``<ul>`` parent:
+
+>>> xp("//ul/li[1]")
+['<li>1</li>', '<li>4</li>']
+
+And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document:
+
+>>> xp("(//ul/li)[1]")
+['<li>1</li>']
+
+Using text nodes in a condition
+-------------------------------
+
+When you need to use the text content as argument to an `XPath string function`_,
+avoid using ``.//text()`` and use just ``.`` instead.
+
+This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
+And when a node-set is converted to a string, which happens when it is passed as argument to
+a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
+
+Example:
+
+>>> from scrapy import Selector
+>>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
+
+Converting a *node-set* to string:
+
+>>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+['Click here to go to the ', 'Next Page']
+>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+['Click here to go to the ']
+
+A *node* converted to a string, however, puts together the text of itself plus of all its descendants:
+
+>>> sel.xpath("//a[1]").getall() # select the first node
+['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+>>> sel.xpath("string(//a[1])").getall() # convert it to string
+['Click here to go to the Next Page']
+
+So, using the ``.//text()`` node-set won't select anything in this case:
+
+>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+[]
+
+But using the ``.`` to mean the node, works:
+
+>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+
+.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
+
.. _topics-selectors-xpath-variables:
Variables in XPath expressions
@@ -295,17 +626,17 @@ some arguments in your queries with placeholders like ``?``,
which are then substituted with values passed with the query.
Here's an example to match an element based on its "id" attribute value,
-without hard-coding it (that was shown previously)::
+without hard-coding it (that was shown previously):
- >>> # `$val` used in the expression, a `val` argument needs to be passed
- >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
- u'Name: My image 1 '
+>>> # `$val` used in the expression, a `val` argument needs to be passed
+>>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
+'Name: My image 1 '
Here's another example, to find the "id" attribute of a ``<div>`` tag containing
-five ``<a>`` children (here we pass the value ``5`` as an integer)::
+five ``<a>`` children (here we pass the value ``5`` as an integer):
- >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
- u'images'
+>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
+'images'
All variable references must have a binding value when calling ``.xpath()``
(otherwise you'll get a ``ValueError: XPath error:`` exception).
@@ -314,13 +645,78 @@ This is done by passing as many named arguments as necessary.
`parsel`_, the library powering Scrapy selectors, has more details and examples
on `XPath variables`_.
-.. _parsel: https://parsel.readthedocs.io/
.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
+
+.. _removing-namespaces:
+
+Removing namespaces
+-------------------
+
+When dealing with scraping projects, it is often quite convenient to get rid of
+namespaces altogether and just work with element names, to write more
+simple/convenient XPaths. You can use the
+:meth:`Selector.remove_namespaces` method for that.
+
+Let's show an example that illustrates this with the Python Insider blog atom feed.
+
+.. highlight:: sh
+
+First, we open the shell with the url we want to scrape::
+
+ $ scrapy shell https://feeds.feedburner.com/PythonInsider
+
+This is how the file starts::
+
+ <?xml version="1.0" encoding="UTF-8"?>
+ <?xml-stylesheet ...
+ <feed xmlns="http://www.w3.org/2005/Atom"
+ xmlns:openSearch="http://a9.com/-/spec/opensearchrss/1.0/"
+ xmlns:blogger="http://schemas.google.com/blogger/2008"
+ xmlns:georss="http://www.georss.org/georss"
+ xmlns:gd="http://schemas.google.com/g/2005"
+ xmlns:thr="http://purl.org/syndication/thread/1.0"
+ xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
+ ...
+
+You can see several namespace declarations including a default
+"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for
+"http://schemas.google.com/g/2005".
+
+.. highlight:: python
+
+Once in the shell we can try selecting all ``<link>`` objects and see that it
+doesn't work (because the Atom XML namespace is obfuscating those nodes):
+
+>>> response.xpath("//link")
+[]
+
+But once we call the :meth:`Selector.remove_namespaces` method, all
+nodes can be accessed directly by their names:
+
+>>> response.selector.remove_namespaces()
+>>> response.xpath("//link")
+[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
+ <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
+ ...
+
+If you wonder why the namespace removal procedure isn't always called by default
+instead of having to call it manually, this is because of two reasons, which, in order
+of relevance, are:
+
+1. Removing namespaces requires to iterate and modify all nodes in the
+ document, which is a reasonably expensive operation to perform by default
+ for all documents crawled by Scrapy
+
+2. There could be some cases where using namespaces is actually required, in
+ case some element names clash between namespaces. These cases are very rare
+ though.
+
+
Using EXSLT extensions
----------------------
-Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions
+Being built atop `lxml`_, Scrapy selectors support some `EXSLT`_ extensions
and come with these pre-registered namespaces to use in XPath expressions:
@@ -337,26 +733,25 @@ Regular expressions
The ``test()`` function, for example, can prove quite useful when XPath's
``starts-with()`` or ``contains()`` are not sufficient.
-Example selecting links in list item with a "class" attribute ending with a digit::
+Example selecting links in list item with a "class" attribute ending with a digit:
- >>> from scrapy import Selector
- >>> doc = """
- ... <div>
- ... <ul>
- ... <li class="item-0"><a href="link1.html">first item</a></li>
- ... <li class="item-1"><a href="link2.html">second item</a></li>
- ... <li class="item-inactive"><a href="link3.html">third item</a></li>
- ... <li class="item-1"><a href="link4.html">fourth item</a></li>
- ... <li class="item-0"><a href="link5.html">fifth item</a></li>
- ... </ul>
- ... </div>
- ... """
- >>> sel = Selector(text=doc, type="html")
- >>> sel.xpath('//li//@href').extract()
- [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html']
- >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract()
- [u'link1.html', u'link2.html', u'link4.html', u'link5.html']
- >>>
+>>> from scrapy import Selector
+>>> doc = u"""
+... <div>
+... <ul>
+... <li class="item-0"><a href="link1.html">first item</a></li>
+... <li class="item-1"><a href="link2.html">second item</a></li>
+... <li class="item-inactive"><a href="link3.html">third item</a></li>
+... <li class="item-1"><a href="link4.html">fourth item</a></li>
+... <li class="item-0"><a href="link5.html">fifth item</a></li>
+... </ul>
+... </div>
+... """
+>>> sel = Selector(text=doc, type="html")
+>>> sel.xpath('//li//@href').getall()
+['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
+>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
+['link1.html', 'link2.html', 'link4.html', 'link5.html']
.. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module.
@@ -372,7 +767,7 @@ extracting text elements for example.
Example extracting microdata (sample content taken from http://schema.org/Product)
with groups of itemscopes and corresponding itemprops::
- >>> doc = """
+ >>> doc = u"""
... <div itemscope itemtype="http://schema.org/Product">
... <span itemprop="name">Kenmore White 17" Microwave</span>
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
@@ -424,35 +819,34 @@ with groups of itemscopes and corresponding itemprops::
... """
>>> sel = Selector(text=doc, type="html")
>>> for scope in sel.xpath('//div[@itemscope]'):
- ... print "current scope:", scope.xpath('@itemtype').extract()
+ ... print("current scope:", scope.xpath('@itemtype').getall())
... props = scope.xpath('''
... set:difference(./descendant::*/@itemprop,
... .//*[@itemscope]/*/@itemprop)''')
- ... print " properties:", props.extract()
- ... print
+ ... print(" properties: %s" % (props.getall()))
+ ... print("")
- current scope: [u'http://schema.org/Product']
- properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review']
+ current scope: ['http://schema.org/Product']
+ properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review']
- current scope: [u'http://schema.org/AggregateRating']
- properties: [u'ratingValue', u'reviewCount']
+ current scope: ['http://schema.org/AggregateRating']
+ properties: ['ratingValue', 'reviewCount']
- current scope: [u'http://schema.org/Offer']
- properties: [u'price', u'availability']
+ current scope: ['http://schema.org/Offer']
+ properties: ['price', 'availability']
- current scope: [u'http://schema.org/Review']
- properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+ current scope: ['http://schema.org/Review']
+ properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
- current scope: [u'http://schema.org/Rating']
- properties: [u'worstRating', u'ratingValue', u'bestRating']
+ current scope: ['http://schema.org/Rating']
+ properties: ['worstRating', 'ratingValue', 'bestRating']
- current scope: [u'http://schema.org/Review']
- properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description']
+ current scope: ['http://schema.org/Review']
+ properties: ['name', 'author', 'datePublished', 'reviewRating', 'description']
- current scope: [u'http://schema.org/Rating']
- properties: [u'worstRating', u'ratingValue', u'bestRating']
+ current scope: ['http://schema.org/Rating']
+ properties: ['worstRating', 'ratingValue', 'bestRating']
- >>>
Here we first iterate over ``itemscope`` elements, and for each one,
we look for all ``itemprops`` elements and exclude those that are themselves
@@ -462,127 +856,44 @@ inside another ``itemscope``.
.. _regular expressions: http://exslt.org/regexp/index.html
.. _set manipulation: http://exslt.org/set/index.html
+Other XPath extensions
+----------------------
-Some XPath tips
----------------
+Scrapy selectors also provide a sorely missed XPath extension function
+``has-class`` that returns ``True`` for nodes that have all of the specified
+HTML classes.
-Here are some tips that you may find useful when using XPath
-with Scrapy selectors, based on `this post from ScrapingHub's blog`_.
-If you are not much familiar with XPath yet,
-you may want to take a look first at this `XPath tutorial`_.
+.. highlight:: html
+For the following HTML::
-.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html
-.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/
+ <p class="foo bar-baz">First</p>
+ <p class="foo">Second</p>
+ <p class="bar">Third</p>
+ <p>Fourth</p>
+.. highlight:: python
-Using text nodes in a condition
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You can use it like this:
-When you need to use the text content as argument to an `XPath string function`_,
-avoid using ``.//text()`` and use just ``.`` instead.
+>>> response.xpath('//p[has-class("foo")]')
+[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+ <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+>>> response.xpath('//p[has-class("foo", "bar-baz")]')
+[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+>>> response.xpath('//p[has-class("foo", "bar")]')
+[]
-This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*.
-And when a node-set is converted to a string, which happens when it is passed as argument to
-a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
+So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
+``p.foo.bar-baz``. Please note, that it is slower in most of the cases,
+because it's a pure-Python function that's invoked for every node in question
+whereas the CSS lookup is translated into XPath and thus runs more efficiently,
+so performance-wise its uses are limited to situations that are not easily
+described with CSS selectors.
-Example::
+Parsel also simplifies adding your own XPath extensions.
- >>> from scrapy import Selector
- >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
-
-Converting a *node-set* to string::
-
- >>> sel.xpath('//a//text()').extract() # take a peek at the node-set
- [u'Click here to go to the ', u'Next Page']
- >>> sel.xpath("string(//a[1]//text())").extract() # convert it to string
- [u'Click here to go to the ']
-
-A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
-
- >>> sel.xpath("//a[1]").extract() # select the first node
- [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
- >>> sel.xpath("string(//a[1])").extract() # convert it to string
- [u'Click here to go to the Next Page']
-
-So, using the ``.//text()`` node-set won't select anything in this case::
-
- >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").extract()
- []
-
-But using the ``.`` to mean the node, works::
-
- >>> sel.xpath("//a[contains(., 'Next Page')]").extract()
- [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-
-.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
-
-Beware of the difference between //node[1] and (//node)[1]
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-``//node[1]`` selects all the nodes occurring first under their respective parents.
-
-``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
-
-Example::
-
- >>> from scrapy import Selector
- >>> sel = Selector(text="""
- ....: <ul class="list">
- ....: <li>1</li>
- ....: <li>2</li>
- ....: <li>3</li>
- ....: </ul>
- ....: <ul class="list">
- ....: <li>4</li>
- ....: <li>5</li>
- ....: <li>6</li>
- ....: </ul>""")
- >>> xp = lambda x: sel.xpath(x).extract()
-
-This gets all first ``<li>`` elements under whatever it is its parent::
-
- >>> xp("//li[1]")
- [u'<li>1</li>', u'<li>4</li>']
-
-And this gets the first ``<li>`` element in the whole document::
-
- >>> xp("(//li)[1]")
- [u'<li>1</li>']
-
-This gets all first ``<li>`` elements under an ``<ul>`` parent::
-
- >>> xp("//ul/li[1]")
- [u'<li>1</li>', u'<li>4</li>']
-
-And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document::
-
- >>> xp("(//ul/li)[1]")
- [u'<li>1</li>']
-
-When querying by class, consider using CSS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Because an element can contain multiple CSS classes, the XPath way to select elements
-by class is the rather verbose::
-
- *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')]
-
-If you use ``@class='someclass'`` you may end up missing elements that have
-other classes, and if you just use ``contains(@class, 'someclass')`` to make up
-for that you may end up with more elements that you want, if they have a different
-class name that shares the string ``someclass``.
-
-As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
-you can just select by class using CSS and then switch to XPath when needed::
-
- >>> from scrapy import Selector
- >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
- >>> sel.css('.shout').xpath('./time/@datetime').extract()
- [u'2014-07-23 19:00']
-
-This is cleaner than using the verbose XPath trick shown above. Just remember
-to use the ``.`` in the XPath expressions that will follow.
+.. autofunction:: parsel.xpathfuncs.set_xpathfunc
.. _topics-selectors-ref:
@@ -596,132 +907,79 @@ Built-in Selectors reference
Selector objects
----------------
-.. class:: Selector(response=None, text=None, type=None)
+.. autoclass:: Selector
- An instance of :class:`Selector` is a wrapper over response to select
- certain parts of its content.
-
- ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
- :class:`~scrapy.http.XmlResponse` object that will be used for selecting and
- extracting data.
-
- ``text`` is a unicode string or utf-8 encoded text for cases when a
- ``response`` isn't available. Using ``text`` and ``response`` together is
- undefined behavior.
-
- ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
-
- If ``type`` is ``None``, the selector automatically chooses the best type
- based on ``response`` type (see below), or defaults to ``"html"`` in case it
- is used together with ``text``.
-
- If ``type`` is ``None`` and a ``response`` is passed, the selector type is
- inferred from the response type as follows:
-
- * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
- * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
- * ``"html"`` for anything else
-
- Otherwise, if ``type`` is set, the selector type will be forced and no
- detection will occur.
-
- .. method:: xpath(query)
-
- Find nodes matching the xpath ``query`` and return the result as a
- :class:`SelectorList` instance with all elements flattened. List
- elements implement :class:`Selector` interface too.
-
- ``query`` is a string containing the XPATH query to apply.
+ .. automethod:: xpath
.. note::
For convenience, this method can be called as ``response.xpath()``
- .. method:: css(query)
-
- Apply the given CSS selector and return a :class:`SelectorList` instance.
-
- ``query`` is a string containing the CSS selector to apply.
-
- In the background, CSS queries are translated into XPath queries using
- `cssselect`_ library and run ``.xpath()`` method.
+ .. automethod:: css
.. note::
- For convenience this method can be called as ``response.css()``
+ For convenience, this method can be called as ``response.css()``
- .. method:: extract()
+ .. automethod:: get
- Serialize and return the matched nodes as a list of unicode strings.
- Percent encoded content is unquoted.
+ See also: :ref:`old-extraction-api`
- .. method:: re(regex)
+ .. autoattribute:: attrib
- Apply the given regex and return a list of unicode strings with the
- matches.
+ See also: :ref:`selecting-attributes`.
- ``regex`` can be either a compiled regular expression or a string which
- will be compiled to a regular expression using ``re.compile(regex)``
+ .. automethod:: re
- .. note::
+ .. automethod:: re_first
- Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``<`` and ``&``).
+ .. automethod:: register_namespace
- .. method:: register_namespace(prefix, uri)
+ .. automethod:: remove_namespaces
- Register the given namespace to be used in this :class:`Selector`.
- Without registering namespaces you can't select or extract data from
- non-standard namespaces. See examples below.
+ .. automethod:: __bool__
- .. method:: remove_namespaces()
-
- Remove all namespaces, allowing to traverse the document using
- namespace-less xpaths. See example below.
-
- .. method:: __nonzero__()
-
- Returns ``True`` if there is any real content selected or ``False``
- otherwise. In other words, the boolean value of a :class:`Selector` is
- given by the contents it selects.
+ .. automethod:: getall
+ This method is added to Selector for consistency; it is more useful
+ with SelectorList. See also: :ref:`old-extraction-api`
SelectorList objects
--------------------
-.. class:: SelectorList
+.. autoclass:: SelectorList
- The :class:`SelectorList` class is a subclass of the builtin ``list``
- class, which provides a few additional methods.
+ .. automethod:: xpath
- .. method:: xpath(query)
+ .. automethod:: css
- Call the ``.xpath()`` method for each element in this list and return
- their results flattened as another :class:`SelectorList`.
+ .. automethod:: getall
- ``query`` is the same argument as the one in :meth:`Selector.xpath`
+ See also: :ref:`old-extraction-api`
- .. method:: css(query)
+ .. automethod:: get
- Call the ``.css()`` method for each element in this list and return
- their results flattened as another :class:`SelectorList`.
+ See also: :ref:`old-extraction-api`
- ``query`` is the same argument as the one in :meth:`Selector.css`
+ .. automethod:: re
- .. method:: extract()
+ .. automethod:: re_first
- Call the ``.extract()`` method for each element in this list and return
- their results flattened, as a list of unicode strings.
+ .. autoattribute:: attrib
- .. method:: re()
+ See also: :ref:`selecting-attributes`.
- Call the ``.re()`` method for each element in this list and return
- their results flattened, as a list of unicode strings.
+.. _selector-examples:
+Examples
+========
+
+.. _selector-examples-html:
Selector examples on HTML response
----------------------------------
-Here's a couple of :class:`Selector` examples to illustrate several concepts.
+Here are some :class:`Selector` examples to illustrate several concepts.
In all cases, we assume there is already a :class:`Selector` instantiated with
a :class:`~scrapy.http.HtmlResponse` object like this::
@@ -735,20 +993,22 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
2. Extract the text of all ``<h1>`` elements from an HTML response body,
returning a list of unicode strings::
- sel.xpath("//h1").extract() # this includes the h1 tag
- sel.xpath("//h1/text()").extract() # this excludes the h1 tag
+ sel.xpath("//h1").getall() # this includes the h1 tag
+ sel.xpath("//h1/text()").getall() # this excludes the h1 tag
3. Iterate over all ``<p>`` tags and print their class attribute::
for node in sel.xpath("//p"):
- print node.xpath("@class").extract()
+ print(node.attrib['class'])
+
+
+.. _selector-examples-xml:
Selector examples on XML response
---------------------------------
-Here's a couple of examples to illustrate several concepts. In both cases we
-assume there is already a :class:`Selector` instantiated with an
-:class:`~scrapy.http.XmlResponse` object like this::
+Here are some examples to illustrate concepts for :class:`Selector` objects
+instantiated with an :class:`~scrapy.http.XmlResponse` object::
sel = Selector(xml_response)
@@ -761,53 +1021,6 @@ assume there is already a :class:`Selector` instantiated with an
a namespace::
sel.register_namespace("g", "http://base.google.com/ns/1.0")
- sel.xpath("//g:price").extract()
-
-.. _removing-namespaces:
-
-Removing namespaces
--------------------
-
-When dealing with scraping projects, it is often quite convenient to get rid of
-namespaces altogether and just work with element names, to write more
-simple/convenient XPaths. You can use the
-:meth:`Selector.remove_namespaces` method for that.
-
-Let's show an example that illustrates this with GitHub blog atom feed.
-
-.. highlight:: sh
-
-First, we open the shell with the url we want to scrape::
-
- $ scrapy shell https://github.com/blog.atom
-
-.. highlight:: python
-
-Once in the shell we can try selecting all ``<link>`` objects and see that it
-doesn't work (because the Atom XML namespace is obfuscating those nodes)::
-
- >>> response.xpath("//link")
- []
-
-But once we call the :meth:`Selector.remove_namespaces` method, all
-nodes can be accessed directly by their names::
-
- >>> response.selector.remove_namespaces()
- >>> response.xpath("//link")
- [<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
- <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>,
- ...
-
-If you wonder why the namespace removal procedure isn't always called by default
-instead of having to call it manually, this is because of two reasons, which, in order
-of relevance, are:
-
-1. Removing namespaces requires to iterate and modify all nodes in the
- document, which is a reasonably expensive operation to perform for all
- documents crawled by Scrapy
-
-2. There could be some cases where using namespaces is actually required, in
- case some element names clash between namespaces. These cases are very rare
- though.
+ sel.xpath("//g:price").getall()
.. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 076dc6bfd..4b770d249 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -30,6 +30,8 @@ Python `import search path`_.
.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
+.. _populating-settings:
+
Populating the settings
=======================
@@ -158,6 +160,27 @@ to any particular component. In that case the module of that component will be
shown, typically an extension, middleware or pipeline. It also means that the
component must be enabled in order for the setting to have any effect.
+.. setting:: ASYNCIO_REACTOR
+
+ASYNCIO_REACTOR
+---------------
+
+Default: ``False``
+
+Whether to install and require the Twisted reactor that uses the asyncio loop.
+
+When this option is set to ``True``, Scrapy will require
+:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. It will
+install this reactor if no reactor is installed yet, such as when using the
+``scrapy`` script or :class:`~scrapy.crawler.CrawlerProcess`. If you are using
+:class:`~scrapy.crawler.CrawlerRunner`, you need to install the correct reactor
+manually. If a different reactor is installed outside Scrapy, it will raise an
+exception.
+
+The default value for this option is currently ``False`` to maintain backward
+compatibility and avoid possible problems caused by using a different Twisted
+reactor.
+
.. setting:: AWS_ACCESS_KEY_ID
AWS_ACCESS_KEY_ID
@@ -178,35 +201,45 @@ Default: ``None``
The AWS secret key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
-.. setting:: BOT_NAME
+.. setting:: AWS_ENDPOINT_URL
AWS_ENDPOINT_URL
----------------
Default: ``None``
-Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality.
+Endpoint URL used for S3-like storage, for example Minio or s3.scality.
-.. setting:: AWS_ENDPOINT_URL
+.. setting:: AWS_USE_SSL
AWS_USE_SSL
-----------
Default: ``None``
-Use this option if you want to disable SSL connection for communication with S3 or S3-like storage.
-By default SSL will be used.
+Use this option if you want to disable SSL connection for communication with
+S3 or S3-like storage. By default SSL will be used.
-.. setting:: AWS_USE_SSL
+.. setting:: AWS_VERIFY
AWS_VERIFY
----------
Default: ``None``
-Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur.
+Verify SSL connection between Scrapy and S3 or S3-like storage. By default
+SSL verification will occur.
-.. setting:: AWS_VERIFY
+.. setting:: AWS_REGION_NAME
+
+AWS_REGION_NAME
+---------------
+
+Default: ``None``
+
+The name of the region associated with the AWS client.
+
+.. setting:: BOT_NAME
BOT_NAME
--------
@@ -214,8 +247,7 @@ BOT_NAME
Default: ``'scrapybot'``
The name of the bot implemented by this Scrapy project (also known as the
-project name). This will be used to construct the User-Agent by default, and
-also for logging.
+project name). This name will be used for the logging too.
It's automatically populated with your project name when you create your
project with the :command:`startproject` command.
@@ -318,16 +350,16 @@ Default: ``0``
Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
-An integer that is used to adjust the request priority based on its depth:
+An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
+a :class:`~scrapy.http.Request` based on its depth.
-- if zero (default), no priority adjustment is made from depth
-- **a positive value will decrease the priority, i.e. higher depth
- requests will be processed later** ; this is commonly used when doing
- breadth-first crawls (BFO)
-- a negative value will increase priority, i.e., higher depth requests
- will be processed sooner (DFO)
+The priority of a request is adjusted as follows::
-See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
+ request.priority = request.priority - ( depth * DEPTH_PRIORITY )
+
+As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request
+priority (BFO), while negative values increase request priority (DFO). See
+also :ref:`faq-bfo-dfo`.
.. note::
@@ -335,17 +367,6 @@ See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
other priority settings :setting:`REDIRECT_PRIORITY_ADJUST`
and :setting:`RETRY_PRIORITY_ADJUST`.
-.. setting:: DEPTH_STATS
-
-DEPTH_STATS
------------
-
-Default: ``True``
-
-Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
-
-Whether to collect maximum depth stats.
-
.. setting:: DEPTH_STATS_VERBOSE
DEPTH_STATS_VERBOSE
@@ -376,6 +397,19 @@ Default: ``10000``
DNS in-memory cache size.
+.. setting:: DNS_RESOLVER
+
+DNS_RESOLVER
+------------
+
+Default: ``'scrapy.resolver.CachingThreadedResolver'``
+
+The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver``
+supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting,
+but works only with IPv4 addresses. Scrapy provides an alternative resolver,
+``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
+take the :setting:`DNS_TIMEOUT` setting into account.
+
.. setting:: DNS_TIMEOUT
DNS_TIMEOUT
@@ -436,9 +470,29 @@ or even enable client-side authentication (and various other things).
which uses the platform's certificates to validate remote endpoints.
**This is only available if you use Twisted>=14.0.**
-If you do use a custom ContextFactory, make sure it accepts a ``method``
-parameter at init (this is the ``OpenSSL.SSL`` method mapping
-:setting:`DOWNLOADER_CLIENT_TLS_METHOD`).
+If you do use a custom ContextFactory, make sure its ``__init__`` method
+accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging``
+parameter (``bool``) and a ``tls_ciphers`` parameter (see
+:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`).
+
+.. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS
+
+DOWNLOADER_CLIENT_TLS_CIPHERS
+-----------------------------
+
+Default: ``'DEFAULT'``
+
+Use this setting to customize the TLS/SSL ciphers used by the default
+HTTP/1.1 downloader.
+
+The setting should contain a string in the `OpenSSL cipher list format`_,
+these ciphers will be used as client ciphers. Changing this setting may be
+necessary to access certain HTTPS websites: for example, you may need to use
+``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
+specific cipher that is not included in ``DEFAULT`` if a website requires it.
+
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
@@ -466,6 +520,20 @@ This setting must be one of these string values:
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
or above (Twisted>=14.0 if you can).
+.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+-------------------------------------
+
+Default: ``False``
+
+Setting this to ``True`` will enable DEBUG level messages about TLS connection
+parameters after establishing HTTPS connections. The kind of information logged
+depends on the versions of OpenSSL and pyOpenSSL.
+
+This setting is only used for the default
+:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
+
.. setting:: DOWNLOADER_MIDDLEWARES
DOWNLOADER_MIDDLEWARES
@@ -536,6 +604,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti
When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
per ip address instead of per domain.
+.. _spider-download_delay-attribute:
+
You can also change this setting per spider by setting ``download_delay``
spider attribute.
@@ -597,7 +667,7 @@ The amount of time (in secs) that the downloader will wait before timing out.
DOWNLOAD_MAXSIZE
----------------
-Default: `1073741824` (1024MB)
+Default: ``1073741824`` (1024MB)
The maximum response size (in bytes) that downloader will download.
@@ -618,7 +688,7 @@ If you want to disable it set to 0.
DOWNLOAD_WARNSIZE
-----------------
-Default: `33554432` (32MB)
+Default: ``33554432`` (32MB)
The response size (in bytes) that downloader will start to warn.
@@ -756,6 +826,7 @@ Default: ``True``
Whether or not to use passive mode when initiating FTP transfers.
+.. reqmeta:: ftp_password
.. setting:: FTP_PASSWORD
FTP_PASSWORD
@@ -774,6 +845,7 @@ in ``Request`` meta.
.. _RFC 1635: https://tools.ietf.org/html/rfc1635
+.. reqmeta:: ftp_user
.. setting:: FTP_USER
FTP_USER
@@ -846,7 +918,7 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
-String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available
+String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
placeholders.
.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
@@ -864,6 +936,15 @@ directives.
.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
+.. setting:: LOG_FORMATTER
+
+LOG_FORMATTER
+-------------
+
+Default: :class:`scrapy.logformatter.LogFormatter`
+
+The class to use for :ref:`formatting log messages <custom-log-formats>` for different actions.
+
.. setting:: LOG_LEVEL
LOG_LEVEL
@@ -882,7 +963,7 @@ LOG_STDOUT
Default: ``False``
If ``True``, all standard output (and error) of your process will be redirected
-to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
+to the log. For example if you ``print('hello')`` it will appear in the Scrapy
log.
.. setting:: LOG_SHORT_NAMES
@@ -895,6 +976,16 @@ Default: ``False``
If ``True``, the logs will just contain the root path. If it is set to ``False``
then it displays the component responsible for the log output
+.. setting:: LOGSTATS_INTERVAL
+
+LOGSTATS_INTERVAL
+-----------------
+
+Default: ``60.0``
+
+The interval (in seconds) between each logging printout of the stats
+by :class:`~scrapy.extensions.logstats.LogStats`.
+
.. setting:: MEMDEBUG_ENABLED
MEMDEBUG_ENABLED
@@ -1101,6 +1192,28 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
this option is enabled by default in settings.py file generated
by ``scrapy startproject`` command.
+.. setting:: ROBOTSTXT_PARSER
+
+ROBOTSTXT_PARSER
+----------------
+
+Default: ``'scrapy.robotstxt.ProtegoRobotParser'``
+
+The parser backend to use for parsing ``robots.txt`` files. For more information see
+:ref:`topics-dlmw-robots`.
+
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt file.
+
.. setting:: SCHEDULER
SCHEDULER
@@ -1153,9 +1266,25 @@ Type of in-memory queue used by scheduler. Other available type is:
SCHEDULER_PRIORITY_QUEUE
------------------------
-Default: ``'queuelib.PriorityQueue'``
+Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
-Type of priority queue used by scheduler.
+Type of priority queue used by the scheduler. Another available type is
+``scrapy.pqueues.DownloaderAwarePriorityQueue``.
+``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
+``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
+domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue``
+does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
+
+.. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE
+
+SCRAPER_SLOT_MAX_ACTIVE_SIZE
+----------------------------
+Default: ``5_000_000``
+
+Soft limit (in bytes) for response data being processed.
+
+While the sum of the sizes of all responses being processed is above this value,
+Scrapy does not process new requests.
.. setting:: SPIDER_CONTRACTS
@@ -1180,7 +1309,7 @@ Default::
'scrapy.contracts.default.ScrapesContract': 3,
}
-A dict containing the scrapy contracts enabled by default in Scrapy. You should
+A dict containing the Scrapy contracts enabled by default in Scrapy. You should
never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
instead. For more info see :ref:`topics-contracts`.
@@ -1211,7 +1340,7 @@ SPIDER_LOADER_WARN_ONLY
Default: ``False``
-By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
+By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
it will fail loudly if there is any ``ImportError`` exception.
But you can choose to silence this exception and turn it into a simple
warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
@@ -1354,7 +1483,10 @@ USER_AGENT
Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
-The default User-Agent to use when crawling, unless overridden.
+The default User-Agent to use when crawling, unless overridden. This user agent is
+also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
+if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
+there is no overridding User-Agent header specified for the request.
Settings documented elsewhere:
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 11ab199f2..3cf8311a6 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -31,7 +31,7 @@ for more info.
Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
is unavailable.
-Through scrapy's settings you can configure it to use any one of
+Through Scrapy's settings you can configure it to use any one of
``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
@@ -177,47 +177,46 @@ all start with the ``[s]`` prefix)::
>>>
-After that, we can start playing with the objects::
+After that, we can start playing with the objects:
- >>> response.xpath('//title/text()').extract_first()
- 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
+>>> response.xpath('//title/text()').get()
+'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
- >>> fetch("https://reddit.com")
+>>> fetch("https://reddit.com")
- >>> response.xpath('//title/text()').extract()
- ['reddit: the front page of the internet']
+>>> response.xpath('//title/text()').get()
+'reddit: the front page of the internet'
- >>> request = request.replace(method="POST")
+>>> request = request.replace(method="POST")
- >>> fetch(request)
+>>> fetch(request)
- >>> response.status
- 404
+>>> response.status
+404
- >>> from pprint import pprint
+>>> from pprint import pprint
- >>> pprint(response.headers)
- {'Accept-Ranges': ['bytes'],
- 'Cache-Control': ['max-age=0, must-revalidate'],
- 'Content-Type': ['text/html; charset=UTF-8'],
- 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
- 'Server': ['snooserv'],
- 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
- 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
- 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
- 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
- 'Vary': ['accept-encoding'],
- 'Via': ['1.1 varnish'],
- 'X-Cache': ['MISS'],
- 'X-Cache-Hits': ['0'],
- 'X-Content-Type-Options': ['nosniff'],
- 'X-Frame-Options': ['SAMEORIGIN'],
- 'X-Moose': ['majestic'],
- 'X-Served-By': ['cache-cdg8730-CDG'],
- 'X-Timer': ['S1481214079.394283,VS0,VE159'],
- 'X-Ua-Compatible': ['IE=edge'],
- 'X-Xss-Protection': ['1; mode=block']}
- >>>
+>>> pprint(response.headers)
+{'Accept-Ranges': ['bytes'],
+ 'Cache-Control': ['max-age=0, must-revalidate'],
+ 'Content-Type': ['text/html; charset=UTF-8'],
+ 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
+ 'Server': ['snooserv'],
+ 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+ 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+ 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+ 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
+ 'Vary': ['accept-encoding'],
+ 'Via': ['1.1 varnish'],
+ 'X-Cache': ['MISS'],
+ 'X-Cache-Hits': ['0'],
+ 'X-Content-Type-Options': ['nosniff'],
+ 'X-Frame-Options': ['SAMEORIGIN'],
+ 'X-Moose': ['majestic'],
+ 'X-Served-By': ['cache-cdg8730-CDG'],
+ 'X-Timer': ['S1481214079.394283,VS0,VE159'],
+ 'X-Ua-Compatible': ['IE=edge'],
+ 'X-Xss-Protection': ['1; mode=block']}
.. _topics-shell-inspect-response:
@@ -263,16 +262,16 @@ When you run the spider, you will get something similar to this::
>>> response.url
'http://example.org'
-Then, you can check if the extraction code is working::
+Then, you can check if the extraction code is working:
- >>> response.xpath('//h1[@class="fn"]')
- []
+>>> response.xpath('//h1[@class="fn"]')
+[]
Nope, it doesn't. So you can open the response in your web browser and see if
-it's the response you were expecting::
+it's the response you were expecting:
- >>> view(response)
- True
+>>> view(response)
+True
Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
crawling::
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index cf1588df8..3f29aa323 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti
Deferred signal handlers
========================
-Some signals support returning `Twisted deferreds`_ from their handlers, see
-the :ref:`topics-signals-ref` below to know which ones.
+Some signals support returning :class:`~twisted.internet.defer.Deferred`
+objects from their handlers, see the :ref:`topics-signals-ref` below to know
+which ones.
-.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
.. _topics-signals-ref:
@@ -135,6 +135,29 @@ item_dropped
to be dropped
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
+item_error
+------------
+
+.. signal:: item_error
+.. function:: item_error(item, response, spider, failure)
+
+ Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
+ an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
+
+ This signal supports returning deferreds from their handlers.
+
+ :param item: the item dropped from the :ref:`topics-item-pipeline`
+ :type item: dict or :class:`~scrapy.item.Item` object
+
+ :param response: the response being processed when the exception was raised
+ :type response: :class:`~scrapy.http.Response` object
+
+ :param spider: the spider which raised the exception
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
+ :param failure: the exception raised
+ :type failure: twisted.python.failure.Failure
+
spider_closed
-------------
@@ -213,8 +236,8 @@ spider_error
This signal does not support returning deferreds from their handlers.
- :param failure: the exception raised as a Twisted `Failure`_ object
- :type failure: `Failure`_ object
+ :param failure: the exception raised
+ :type failure: twisted.python.failure.Failure
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
@@ -256,6 +279,22 @@ request_dropped
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
+request_reached_downloader
+---------------------------
+
+.. signal:: request_reached_downloader
+.. function:: request_reached_downloader(request, spider)
+
+ Sent when a :class:`~scrapy.http.Request` reached downloader.
+
+ The signal does not support returning deferreds from their handlers.
+
+ :param request: the request that reached downloader
+ :type request: :class:`~scrapy.http.Request` object
+
+ :param spider: the spider that yielded the request
+ :type spider: :class:`~scrapy.spiders.Spider` object
+
response_received
-----------------
@@ -294,5 +333,3 @@ response_downloaded
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
-
-.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index c297ed556..0e8210130 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied.
If you want to disable a builtin middleware (the ones defined in
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
-in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
+in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
value. For example, if you want to disable the off-site middleware::
SPIDER_MIDDLEWARES = {
@@ -54,11 +54,17 @@ value. For example, if you want to disable the off-site middleware::
Finally, keep in mind that some middlewares may need to be enabled through a
particular setting. See each middleware documentation for more info.
+.. _custom-spider-middleware:
+
Writing your own spider middleware
==================================
-Each middleware component is a Python class that defines one or more of the
-following methods:
+Each spider middleware is a Python class that defines one or more of the
+methods defined below.
+
+The main entry point is the ``from_crawler`` class method, which receives a
+:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
+object gives you access, for example, to the :ref:`settings <topics-settings>`.
.. module:: scrapy.spidermiddlewares
@@ -78,7 +84,8 @@ following methods:
If it raises an exception, Scrapy won't bother calling any other spider
middleware :meth:`process_spider_input` and will call the request
- errback. The output of the errback is chained back in the other
+ errback if there is one, otherwise it will start the :meth:`process_spider_exception`
+ chain. The output of the errback is chained back in the other
direction for :meth:`process_spider_output` to process it, or
:meth:`process_spider_exception` if it raised an exception.
@@ -112,11 +119,11 @@ following methods:
.. method:: process_spider_exception(response, exception, spider)
- This method is called when a spider or :meth:`process_spider_input`
- method (from other spider middleware) raises an exception.
+ This method is called when a spider or :meth:`process_spider_output`
+ method (from a previous spider middleware) raises an exception.
:meth:`process_spider_exception` should return either ``None`` or an
- iterable of :class:`~scrapy.http.Response`, dict or
+ iterable of :class:`~scrapy.http.Request`, dict or
:class:`~scrapy.item.Item` objects.
If it returns ``None``, Scrapy will continue processing this exception,
@@ -125,7 +132,8 @@ following methods:
exception reaches the engine (where it's logged and discarded).
If it returns an iterable the :meth:`process_spider_output` pipeline
- kicks in, and no other :meth:`process_spider_exception` will be called.
+ kicks in, starting from the next spider middleware, and no other
+ :meth:`process_spider_exception` will be called.
:param response: the response being processed when the exception was
raised
@@ -200,7 +208,7 @@ DepthMiddleware
.. class:: DepthMiddleware
DepthMiddleware is used for tracking the depth of each Request inside the
- site being scraped. It works by setting `request.meta['depth'] = 0` whenever
+ site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
there is no value previously set (usually just the first Request) and
incrementing it by 1 otherwise.
@@ -212,7 +220,8 @@ DepthMiddleware
* :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
crawl for any site. If zero, no limit will be imposed.
- * :setting:`DEPTH_STATS` - Whether to collect depth stats.
+ * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of
+ requests for each depth.
* :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
their depth.
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index c2c271245..b0fb14e24 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -72,8 +72,6 @@ scrapy.Spider
spider that crawls ``mywebsite.com`` would often be called
``mywebsite``.
- .. note:: In Python 2 this must be ASCII only.
-
.. attribute:: allowed_domains
An optional list of strings containing domains that this spider is
@@ -88,7 +86,7 @@ scrapy.Spider
A list of URLs where the spider will begin to crawl from, when no
particular URLs are specified. So, the first pages downloaded will be those
- listed here. The subsequent URLs will be generated successively from data
+ listed here. The subsequent :class:`~scrapy.http.Request` will be generated successively from data
contained in the start URLs.
.. attribute:: custom_settings
@@ -129,7 +127,7 @@ scrapy.Spider
You probably won't need to override this directly because the default
implementation acts as a proxy to the :meth:`__init__` method, calling
- it with the given arguments `args` and named arguments `kwargs`.
+ it with the given arguments ``args`` and named arguments ``kwargs``.
Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
attributes in the new instance so they can be accessed later inside the
@@ -190,7 +188,7 @@ scrapy.Spider
.. method:: log(message, [level, component])
Wrapper that sends a log message through the Spider's :attr:`logger`,
- kept for backwards compatibility. For more information see
+ kept for backward compatibility. For more information see
:ref:`topics-logging-from-spiders`.
.. method:: closed(reason)
@@ -229,11 +227,11 @@ Return multiple Requests and items from a single callback::
]
def parse(self, response):
- for h3 in response.xpath('//h3').extract():
+ for h3 in response.xpath('//h3').getall():
yield {"title": h3}
- for url in response.xpath('//a/@href').extract():
- yield scrapy.Request(url, callback=self.parse)
+ for href in response.xpath('//a/@href').getall():
+ yield scrapy.Request(response.urljoin(href), self.parse)
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
to give data more structure you can use :ref:`topics-items`::
@@ -251,11 +249,11 @@ to give data more structure you can use :ref:`topics-items`::
yield scrapy.Request('http://www.example.com/3.html', self.parse)
def parse(self, response):
- for h3 in response.xpath('//h3').extract():
+ for h3 in response.xpath('//h3').getall():
yield MyItem(title=h3)
- for url in response.xpath('//a/@href').extract():
- yield scrapy.Request(url, callback=self.parse)
+ for href in response.xpath('//a/@href').getall():
+ yield scrapy.Request(response.urljoin(href), self.parse)
.. _spiderargs:
@@ -298,13 +296,13 @@ The above example can also be written as follows::
Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
-If you were to set the `start_urls` attribute from the command line,
+If you were to set the ``start_urls`` attribute from the command line,
you would have to parse it on your own into a list
using something like
`ast.literal_eval <https://docs.python.org/library/ast.html#ast.literal_eval>`_
or `json.loads <https://docs.python.org/library/json.html#json.loads>`_
and then set it as an attribute.
-Otherwise, you would cause iteration over a `start_urls` string
+Otherwise, you would cause iteration over a ``start_urls`` string
(a very common python pitfall)
resulting in each character being seen as a separate url.
@@ -374,16 +372,23 @@ CrawlSpider
Crawling rules
~~~~~~~~~~~~~~
-.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
+.. autoclass:: Rule
``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
- defines how links will be extracted from each crawled page.
+ defines how links will be extracted from each crawled page. Each produced link will
+ be used to generate a :class:`~scrapy.http.Request` object, which will contain the
+ link's text in its ``meta`` dictionary (under the ``link_text`` key).
+ If omitted, a default link extractor created with no arguments will be used,
+ resulting in all links being extracted.
``callback`` is a callable or a string (in which case a method from the spider
object with that name will be used) to be called for each link extracted with
- the specified link_extractor. This callback receives a response as its first
- argument and must return a list containing :class:`~scrapy.item.Item` and/or
- :class:`~scrapy.http.Request` objects (or any subclass of them).
+ the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
+ as its first argument and must return either a single instance or an iterable of
+ :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
+ (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
+ object will contain the text of the link that produced the :class:`~scrapy.http.Request`
+ in its ``meta`` dictionary (under the ``link_text`` key)
.. warning:: When writing crawl spider rules, avoid using ``parse`` as
callback, since the :class:`CrawlSpider` uses the ``parse`` method
@@ -402,10 +407,18 @@ Crawling rules
of links extracted from each response using the specified ``link_extractor``.
This is mainly used for filtering purposes.
- ``process_request`` is a callable, or a string (in which case a method from
- the spider object with that name will be used) which will be called with
- every request extracted by this rule, and must return a request or None (to
- filter out the request).
+ ``process_request`` is a callable (or a string, in which case a method from
+ the spider object with that name will be used) which will be called for every
+ :class:`~scrapy.http.Request` extracted by this rule. This callable should
+ take said request as first argument and the :class:`~scrapy.http.Response`
+ from which the request originated as second argument. It must return a
+ ``Request`` object or ``None`` (to filter out the request).
+
+ ``errback`` is a callable or a string (in which case a method from the spider
+ object with that name will be used) to be called if any exception is
+ raised while processing a request generated by the rule.
+ It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
+ instance as first parameter.
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
@@ -434,8 +447,9 @@ Let's now take a look at an example CrawlSpider with rules::
self.logger.info('Hi, this is an item page! %s', response.url)
item = scrapy.Item()
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
- item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
- item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
+ item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
+ item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
+ item['link_text'] = response.meta['link_text']
return item
@@ -545,12 +559,12 @@ These spiders are pretty easy to use, let's have a look at one example::
itertag = 'item'
def parse_node(self, response, node):
- self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
+ self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
item = TestItem()
- item['id'] = node.xpath('@id').extract()
- item['name'] = node.xpath('name').extract()
- item['description'] = node.xpath('description').extract()
+ item['id'] = node.xpath('@id').get()
+ item['name'] = node.xpath('name').get()
+ item['description'] = node.xpath('description').get()
return item
Basically what we did up there was to create a spider that downloads a feed from
@@ -655,7 +669,7 @@ SitemapSpider
.. attribute:: sitemap_follow
- A list of regexes of sitemap that should be followed. This is is only
+ A list of regexes of sitemap that should be followed. This is only
for sites that use `Sitemap index files`_ that point to other sitemap
files.
@@ -680,6 +694,50 @@ SitemapSpider
Default is ``sitemap_alternate_links`` disabled.
+ .. method:: sitemap_filter(entries)
+
+ This is a filter function that could be overridden to select sitemap entries
+ based on their attributes.
+
+ For example::
+
+ <url>
+ <loc>http://example.com/</loc>
+ <lastmod>2005-01-01</lastmod>
+ </url>
+
+ We can define a ``sitemap_filter`` function to filter ``entries`` by date::
+
+ from datetime import datetime
+ from scrapy.spiders import SitemapSpider
+
+ class FilteredSitemapSpider(SitemapSpider):
+ name = 'filtered_sitemap_spider'
+ allowed_domains = ['example.com']
+ sitemap_urls = ['http://example.com/sitemap.xml']
+
+ def sitemap_filter(self, entries):
+ for entry in entries:
+ date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
+ if date_time.year >= 2005:
+ yield entry
+
+ This would retrieve only ``entries`` modified on 2005 and the following
+ years.
+
+ Entries are dict objects extracted from the sitemap document.
+ Usually, the key is the tag name and the value is the text inside it.
+
+ It's important to notice that:
+
+ - as the loc attribute is required, entries without this tag are discarded
+ - alternate links are stored in a list with the key ``alternate``
+ (see ``sitemap_alternate_links``)
+ - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname``
+
+ If you omit this method, all entries found in sitemaps will be
+ processed, observing other attributes and their settings.
+
SitemapSpider examples
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index dd0c6216b..3dd829ebe 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -57,15 +57,15 @@ Set stat value only if lower than previous::
stats.min_value('min_free_memory_percent', value)
-Get stat value::
+Get stat value:
- >>> stats.get_value('custom_count')
- 1
+>>> stats.get_value('custom_count')
+1
-Get all stats::
+Get all stats:
- >>> stats.get_stats()
- {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
+>>> stats.get_stats()
+{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
Available Stats Collectors
==========================
@@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
default Stats Collector used is the :class:`MemoryStatsCollector`.
-.. module:: scrapy.statscollectors
- :synopsis: Stats Collectors
+.. currentmodule:: scrapy.statscollectors
MemoryStatsCollector
--------------------
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index ce79c9f35..47d8d393c 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -1,12 +1,11 @@
+.. currentmodule:: scrapy.extensions.telnet
+
.. _topics-telnetconsole:
==============
Telnet Console
==============
-.. module:: scrapy.extensions.telnet
- :synopsis: The Telnet Console
-
Scrapy comes with a built-in telnet console for inspecting and controlling a
Scrapy running process. The telnet console is just a regular python shell
running inside the Scrapy process, so you can do literally anything from it.
@@ -16,6 +15,17 @@ The telnet console is a :ref:`built-in Scrapy extension
disable it if you want. For more information about the extension itself see
:ref:`topics-extensions-ref-telnetconsole`.
+.. warning::
+ It is not secure to use telnet console via public networks, as telnet
+ doesn't provide any transport-layer security. Having username/password
+ authentication doesn't change that.
+
+ Intended usage is connecting to a running Scrapy spider locally
+ (spider process and telnet client are on the same machine)
+ or over a secure connection (VPN, SSH tunnel).
+ Please avoid using telnet console over insecure connections,
+ or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option.
+
.. highlight:: none
How to access the telnet console
@@ -26,8 +36,26 @@ The telnet console listens in the TCP port defined in the
the console you need to type::
telnet localhost 6023
+ Trying localhost...
+ Connected to localhost.
+ Escape character is '^]'.
+ Username:
+ Password:
>>>
-
+
+By default Username is ``scrapy`` and Password is autogenerated. The
+autogenerated Password can be seen on Scrapy logs like the example below::
+
+ 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
+
+Default Username and Password can be overridden by the settings
+:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
+
+.. warning::
+ Username and password provide only a limited protection, as telnet
+ is not using secure transport - by default traffic is not encrypted
+ even if username and password are set.
+
You need the telnet program which comes installed by default in Windows, and
most Linux distros.
@@ -160,3 +188,24 @@ Default: ``'127.0.0.1'``
The interface the telnet console should listen on
+
+.. setting:: TELNETCONSOLE_USERNAME
+
+TELNETCONSOLE_USERNAME
+----------------------
+
+Default: ``'scrapy'``
+
+The username used for the telnet console
+
+
+.. setting:: TELNETCONSOLE_PASSWORD
+
+TELNETCONSOLE_PASSWORD
+----------------------
+
+Default: ``None``
+
+The password used for the telnet console, default behaviour is to have it
+autogenerated
+
diff --git a/docs/topics/ubuntu.rst b/docs/topics/ubuntu.rst
deleted file mode 100644
index 81ce800aa..000000000
--- a/docs/topics/ubuntu.rst
+++ /dev/null
@@ -1,41 +0,0 @@
-:orphan: Ubuntu packages are obsolete
-
-.. _topics-ubuntu:
-
-===============
-Ubuntu packages
-===============
-
-.. versionadded:: 0.10
-
-`Scrapinghub`_ publishes apt-gettable packages which are generally fresher than
-those in Ubuntu, and more stable too since they're continuously built from
-`GitHub repo`_ (master & stable branches) and so they contain the latest bug
-fixes.
-
-.. caution:: These packages are currently not updated and may not work on
- Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`.
-
-To use the packages:
-
-1. Import the GPG key used to sign Scrapy packages into APT keyring::
-
- sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
-
-2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command::
-
- echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list
-
-3. Update package lists and install the scrapy package:
-
- .. parsed-literal::
-
- sudo apt-get update && sudo apt-get install scrapy
-
-.. note:: Repeat step 3 if you are trying to upgrade Scrapy.
-
-.. warning:: `python-scrapy` is a different package provided by official debian
- repositories, it's very outdated and it isn't supported by Scrapy team.
-
-.. _Scrapinghub: https://scrapinghub.com/
-.. _GitHub repo: https://github.com/scrapy/scrapy
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 0421ba544..227085f02 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
* *A* is the major version. This will rarely change and will signify very
large changes.
* *B* is the release number. This will include many changes including features
- and things that possibly break backwards compatibility, although we strive to
+ and things that possibly break backward compatibility, although we strive to
keep theses cases at a minimum.
* *C* is the bugfix release number.
diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index 3bef20bf3..da7a0022b 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python
-from __future__ import print_function
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion
index 564991aa8..e995947cb 100644
--- a/extras/scrapy_zsh_completion
+++ b/extras/scrapy_zsh_completion
@@ -1,25 +1,210 @@
#compdef scrapy
-
-# zsh completion for the Scrapy command-line tool
-
_scrapy() {
- local curcontext="$curcontext" cmd spiders
+ local context state state_descr line
typeset -A opt_args
- cmd=$words[2]
-
- case "$cmd" in
- crawl|edit|check)
- spiders=$(scrapy list 2>/dev/null) || spiders=""
- if [[ -n "$spiders" ]]; then
- compadd `echo $spiders`
- fi
- ;;
- *)
- if [[ CURRENT -eq 2 ]]; then
- _arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
- fi
- ;;
+ _arguments \
+ "(- 1 *)--help[Help]" \
+ "1: :->command" \
+ "*:: :->args"
+
+ case $state in
+ command)
+ _scrapy_cmds
+ ;;
+ args)
+ case $words[1] in
+ bench)
+ _scrapy_glb_opts
+ ;;
+ fetch)
+ local options=(
+ '--headers[print response HTTP headers instead of body]'
+ '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+ '--spider[use this spider]:spider:_scrapy_spiders'
+ '1::URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ genspider)
+ local options=(
+ {-l,--list}'[List available templates]'
+ {-e,--edit}'[Edit spider after creating it]'
+ '--force[If the spider already exists, overwrite it with the template]'
+ {-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
+ {-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
+ '1:name:(NAME)'
+ '2:domain:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ runspider)
+ local options=(
+ {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+ {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+ '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+ '1:spider file:_files -g \*.py'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ settings)
+ local options=(
+ '--get=[print raw setting value]:option:(SETTING)'
+ '--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
+ '--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
+ '--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
+ '--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ shell)
+ local options=(
+ '-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
+ '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+ '--spider[use this spider]:spider:_scrapy_spiders'
+ '::file:_files -g \*.html'
+ '::URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ startproject)
+ local options=(
+ '1:name:(NAME)'
+ '2:dir:_dir_list'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ version)
+ local options=(
+ {-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ view)
+ local options=(
+ '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+ '--spider[use this spider]:spider:_scrapy_spiders'
+ '1:URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ check)
+ local options=(
+ '(- 1 *)'{-l,--list}'[only list contracts, without checking them]'
+ {-v,--verbose}'[print contract tests for all spiders]'
+ '1:spider:_scrapy_spiders'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ crawl)
+ local options=(
+ {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+ {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)'
+ '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+ '1:spider:_scrapy_spiders'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ edit)
+ local options=(
+ '1:spider:_scrapy_spiders'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ list)
+ _scrapy_glb_opts
+ ;;
+ parse)
+ local options=(
+ '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+ '--spider[use this spider without looking for one]:spider:_scrapy_spiders'
+ '--pipelines[process items through pipelines]'
+ "--nolinks[don't show links to follow (extracted requests)]"
+ "--noitems[don't show scraped items]"
+ '--nocolour[avoid using pygments to colorize the output]'
+ {-r,--rules}'[use CrawlSpider rules to discover the callback]'
+ {-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
+ {-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
+ '--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
+ {-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
+ {-v,--verbose}'[print each depth level one by one]'
+ '1:URL:_httpie_urls'
+ )
+ _scrapy_glb_opts $options
+ ;;
+ esac
+ ;;
esac
}
-_scrapy
\ No newline at end of file
+_scrapy_cmds() {
+ local -a commands project_commands
+ commands=(
+ 'bench:Run quick benchmark test'
+ 'fetch:Fetch a URL using the Scrapy downloader'
+ 'genspider:Generate new spider using pre-defined templates'
+ 'runspider:Run a self-contained spider (without creating a project)'
+ 'settings:Get settings values'
+ 'shell:Interactive scraping console'
+ 'startproject:Create new project'
+ 'version:Print Scrapy version'
+ 'view:Open URL in browser, as seen by Scrapy'
+ )
+ project_commands=(
+ 'check:Check spider contracts'
+ 'crawl:Run a spider'
+ 'edit:Edit spider'
+ 'list:List available spiders'
+ 'parse:Parse URL (using its spider) and print the results'
+ )
+ if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+ commands=(${commands[@]} ${project_commands[@]})
+ fi
+ _describe -t common-commands 'common commands' commands
+}
+
+_scrapy_glb_opts() {
+ local -a options
+ options=(
+ '(- *)'{-h,--help}'[show this help message and exit]'
+ '(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
+ '--pidfile=[write process ID to FILE]:file:_files'
+ '--profile=[write python cProfile stats to FILE]:file:_files'
+ '(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
+ '(-L --loglevel --logfile)--nolog[disable logging completely]'
+ '--pdb[enable pdb on failure]'
+ '*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
+ )
+ options=(${options[@]} "$@")
+ _arguments $options
+}
+
+_httpie_urls() {
+
+ local ret=1
+
+ if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
+ local expl
+ compset -S '[^:/]*' && compstate[to_end]=''
+ _wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
+ else
+ _urls && ret=0
+ fi
+
+ return $ret
+
+}
+
+_scrapy_spiders() {
+
+ local ret=1
+
+ if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+ compadd -S '' $(scrapy list) && ret=0
+ else
+ compadd -S '' SPIDER && ret=0
+ fi
+
+ return $ret
+}
+
+_scrapy $@
diff --git a/pytest.ini b/pytest.ini
index 73d169601..bae68cd3a 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -2,5 +2,252 @@
usefixtures = chdir
python_files=test_*.py __init__.py
python_classes=
-addopts = --doctest-modules --assert=plain
+addopts =
+ --assert=plain
+ --doctest-modules
+ --ignore=docs/_ext
+ --ignore=docs/conf.py
+ --ignore=docs/news.rst
+ --ignore=docs/topics/dynamic-content.rst
+ --ignore=docs/topics/items.rst
+ --ignore=docs/topics/leaks.rst
+ --ignore=docs/topics/loaders.rst
+ --ignore=docs/topics/selectors.rst
+ --ignore=docs/topics/shell.rst
+ --ignore=docs/topics/stats.rst
+ --ignore=docs/topics/telnetconsole.rst
+ --ignore=docs/utils
twisted = 1
+markers =
+ only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+flake8-ignore =
+ # Files that are only meant to provide top-level imports are expected not
+ # to use any of their imports:
+ scrapy/core/downloader/handlers/http.py F401
+ scrapy/http/__init__.py F401
+ # Issues pending a review:
+ # extras
+ extras/qps-bench-server.py E501
+ extras/qpsclient.py E501 E501
+ # scrapy/commands
+ scrapy/commands/__init__.py E128 E501
+ scrapy/commands/check.py E501
+ scrapy/commands/crawl.py E501
+ scrapy/commands/edit.py E501
+ scrapy/commands/fetch.py E401 E501 E128 E731
+ scrapy/commands/genspider.py E128 E501 E502
+ scrapy/commands/parse.py E128 E501 E731 E226
+ scrapy/commands/runspider.py E501
+ scrapy/commands/settings.py E128
+ scrapy/commands/shell.py E128 E501 E502
+ scrapy/commands/startproject.py E127 E501 E128
+ scrapy/commands/version.py E501 E128
+ # scrapy/contracts
+ scrapy/contracts/__init__.py E501 W504
+ scrapy/contracts/default.py E128
+ # scrapy/core
+ scrapy/core/engine.py E501 E128 E127 E306 E502
+ scrapy/core/scheduler.py E501
+ scrapy/core/scraper.py E501 E306 E128 W504
+ scrapy/core/spidermw.py E501 E731 E126 E226
+ scrapy/core/downloader/__init__.py E501
+ scrapy/core/downloader/contextfactory.py E501 E128 E126
+ scrapy/core/downloader/middleware.py E501 E502
+ scrapy/core/downloader/tls.py E501 E305 E241
+ scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
+ scrapy/core/downloader/handlers/__init__.py E501
+ scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
+ scrapy/core/downloader/handlers/http10.py E501
+ scrapy/core/downloader/handlers/http11.py E501
+ scrapy/core/downloader/handlers/s3.py E501 E128 E126
+ # scrapy/downloadermiddlewares
+ scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
+ scrapy/downloadermiddlewares/decompression.py E501
+ scrapy/downloadermiddlewares/defaultheaders.py E501
+ scrapy/downloadermiddlewares/httpcache.py E501 E126
+ scrapy/downloadermiddlewares/httpcompression.py E501 E128
+ scrapy/downloadermiddlewares/httpproxy.py E501
+ scrapy/downloadermiddlewares/redirect.py E501 W504
+ scrapy/downloadermiddlewares/retry.py E501 E126
+ scrapy/downloadermiddlewares/robotstxt.py E501
+ scrapy/downloadermiddlewares/stats.py E501
+ # scrapy/extensions
+ scrapy/extensions/closespider.py E501 E128 E123
+ scrapy/extensions/corestats.py E501
+ scrapy/extensions/feedexport.py E128 E501
+ scrapy/extensions/httpcache.py E128 E501 E303
+ scrapy/extensions/memdebug.py E501
+ scrapy/extensions/spiderstate.py E501
+ scrapy/extensions/telnet.py E501 W504
+ scrapy/extensions/throttle.py E501
+ # scrapy/http
+ scrapy/http/common.py E501
+ scrapy/http/cookies.py E501
+ scrapy/http/request/__init__.py E501
+ scrapy/http/request/form.py E501 E123
+ scrapy/http/request/json_request.py E501
+ scrapy/http/response/__init__.py E501 E128
+ scrapy/http/response/text.py E501 E128 E124
+ # scrapy/linkextractors
+ scrapy/linkextractors/__init__.py E731 E501 E402 W504
+ scrapy/linkextractors/lxmlhtml.py E501 E731 E226
+ # scrapy/loader
+ scrapy/loader/__init__.py E501 E128
+ scrapy/loader/processors.py E501
+ # scrapy/pipelines
+ scrapy/pipelines/__init__.py E501
+ scrapy/pipelines/files.py E116 E501 E266
+ scrapy/pipelines/images.py E265 E501
+ scrapy/pipelines/media.py E125 E501 E266
+ # scrapy/selector
+ scrapy/selector/__init__.py F403
+ scrapy/selector/unified.py E501 E111
+ # scrapy/settings
+ scrapy/settings/__init__.py E501
+ scrapy/settings/default_settings.py E501 E114 E116 E226
+ scrapy/settings/deprecated.py E501
+ # scrapy/spidermiddlewares
+ scrapy/spidermiddlewares/httperror.py E501
+ scrapy/spidermiddlewares/offsite.py E501
+ scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
+ scrapy/spidermiddlewares/urllength.py E501
+ # scrapy/spiders
+ scrapy/spiders/__init__.py E501 E402
+ scrapy/spiders/crawl.py E501
+ scrapy/spiders/feed.py E501
+ scrapy/spiders/sitemap.py E501
+ # scrapy/utils
+ scrapy/utils/asyncio.py E501
+ scrapy/utils/benchserver.py E501
+ scrapy/utils/conf.py E402 E501
+ scrapy/utils/console.py E306 E305
+ scrapy/utils/datatypes.py E501 E226
+ scrapy/utils/decorators.py E501
+ scrapy/utils/defer.py E501 E128
+ scrapy/utils/deprecate.py E128 E501 E127 E502
+ scrapy/utils/gz.py E305 E501 W504
+ scrapy/utils/http.py F403 E226
+ scrapy/utils/httpobj.py E501
+ scrapy/utils/iterators.py E501 E701
+ scrapy/utils/log.py E128 W503
+ scrapy/utils/markup.py F403
+ scrapy/utils/misc.py E501 E226
+ scrapy/utils/multipart.py F403
+ scrapy/utils/project.py E501
+ scrapy/utils/python.py E501
+ scrapy/utils/reactor.py E226
+ scrapy/utils/reqser.py E501
+ scrapy/utils/request.py E127 E501
+ scrapy/utils/response.py E501 E128
+ scrapy/utils/signal.py E501 E128
+ scrapy/utils/sitemap.py E501
+ scrapy/utils/spider.py E271 E501
+ scrapy/utils/ssl.py E501
+ scrapy/utils/test.py E501
+ scrapy/utils/url.py E501 F403 E128 F405
+ # scrapy
+ scrapy/__init__.py E402 E501
+ scrapy/cmdline.py E501
+ scrapy/crawler.py E501
+ scrapy/dupefilters.py E501 E202
+ scrapy/exceptions.py E501
+ scrapy/exporters.py E501 E226
+ scrapy/interfaces.py E501
+ scrapy/item.py E501 E128
+ scrapy/link.py E501
+ scrapy/logformatter.py E501
+ scrapy/mail.py E402 E128 E501 E502
+ scrapy/middleware.py E128 E501
+ scrapy/pqueues.py E501
+ scrapy/resolver.py E501
+ scrapy/responsetypes.py E128 E501 E305
+ scrapy/robotstxt.py E501
+ scrapy/shell.py E501
+ scrapy/signalmanager.py E501
+ scrapy/spiderloader.py E225 F841 E501 E126
+ scrapy/squeues.py E128
+ scrapy/statscollectors.py E501
+ # tests
+ tests/__init__.py E402 E501
+ tests/mockserver.py E401 E501 E126 E123
+ tests/pipelines.py F841 E226
+ tests/spiders.py E501 E127
+ tests/test_closespider.py E501 E127
+ tests/test_command_fetch.py E501
+ tests/test_command_parse.py E501 E128 E303 E226
+ tests/test_command_shell.py E501 E128
+ tests/test_commands.py E128 E501
+ tests/test_contracts.py E501 E128
+ tests/test_crawl.py E501 E741 E265
+ tests/test_crawler.py F841 E306 E501
+ tests/test_dependencies.py F841 E501 E305
+ tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
+ tests/test_downloadermiddleware.py E501
+ tests/test_downloadermiddleware_ajaxcrawlable.py E501
+ tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
+ tests/test_downloadermiddleware_decompression.py E127
+ tests/test_downloadermiddleware_defaultheaders.py E501
+ tests/test_downloadermiddleware_downloadtimeout.py E501
+ tests/test_downloadermiddleware_httpcache.py E501 E305
+ tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
+ tests/test_downloadermiddleware_httpproxy.py E501 E128
+ tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
+ tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
+ tests/test_downloadermiddleware_robotstxt.py E501
+ tests/test_downloadermiddleware_stats.py E501
+ tests/test_dupefilters.py E221 E501 E741 E128 E124
+ tests/test_engine.py E401 E501 E128
+ tests/test_exporters.py E501 E731 E306 E128 E124
+ tests/test_extension_telnet.py F841
+ tests/test_feedexport.py E501 F841 E241
+ tests/test_http_cookies.py E501
+ tests/test_http_headers.py E501
+ tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
+ tests/test_http_response.py E501 E301 E128 E265
+ tests/test_item.py E701 E128 F841 E306
+ tests/test_link.py E501
+ tests/test_linkextractors.py E501 E128 E124
+ tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
+ tests/test_logformatter.py E128 E501 E122
+ tests/test_mail.py E128 E501 E305
+ tests/test_middleware.py E501 E128
+ tests/test_pipeline_crawl.py E131 E501 E128 E126
+ tests/test_pipeline_files.py E501 E303 E272 E226
+ tests/test_pipeline_images.py F841 E501 E303
+ tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
+ tests/test_proxy_connect.py E501 E741
+ tests/test_request_cb_kwargs.py E501
+ tests/test_responsetypes.py E501 E305
+ tests/test_robotstxt_interface.py E501 E501
+ tests/test_scheduler.py E501 E126 E123
+ tests/test_selector.py E501 E127
+ tests/test_spider.py E501
+ tests/test_spidermiddleware.py E501 E226
+ tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
+ tests/test_spidermiddleware_offsite.py E501 E128 E111
+ tests/test_spidermiddleware_output_chain.py E501 E226
+ tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
+ tests/test_squeues.py E501 E701 E741
+ tests/test_utils_asyncio.py E501
+ tests/test_utils_conf.py E501 E303 E128
+ tests/test_utils_curl.py E501
+ tests/test_utils_datatypes.py E402 E501 E305
+ tests/test_utils_defer.py E306 E501 F841 E226
+ tests/test_utils_deprecate.py F841 E306 E501
+ tests/test_utils_http.py E501 E128 W504
+ tests/test_utils_iterators.py E501 E128 E129 E303 E241
+ tests/test_utils_log.py E741 E226
+ tests/test_utils_python.py E501 E303 E731 E701 E305
+ tests/test_utils_reqser.py E501 E128
+ tests/test_utils_request.py E501 E128 E305
+ tests/test_utils_response.py E501
+ tests/test_utils_signal.py E741 F841 E731 E226
+ tests/test_utils_sitemap.py E128 E501 E124
+ tests/test_utils_spider.py E305
+ tests/test_utils_template.py E305
+ tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
+ tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
+ tests/test_cmdline/__init__.py E501
+ tests/test_settings/__init__.py E501 E128
+ tests/test_spiderloader/__init__.py E128 E501
+ tests/test_utils_misc/__init__.py E501
diff --git a/requirements-py3.txt b/requirements-py3.txt
deleted file mode 100644
index 2aae3ae65..000000000
--- a/requirements-py3.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-Twisted >= 17.9.0
-lxml>=3.2.4
-pyOpenSSL>=0.13.1
-cssselect>=0.9
-queuelib>=1.1.1
-w3lib>=1.17.0
-service_identity
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 2a94d742d..000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-Twisted>=13.1.0
-lxml
-pyOpenSSL
-cssselect>=0.9
-w3lib>=1.17.0
-queuelib
-six>=1.5.2
-PyDispatcher>=2.0.5
-service_identity
-parsel>=1.4
diff --git a/scrapy/VERSION b/scrapy/VERSION
index bc80560fa..27f9cd322 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-1.5.0
+1.8.0
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 03ec6c667..fb8357f3c 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -14,8 +14,8 @@ del pkgutil
# Check minimum required Python version
import sys
-if sys.version_info < (2, 7):
- print("Scrapy %s requires Python 2.7" % __version__)
+if sys.version_info < (3, 5):
+ print("Scrapy %s requires Python 3.5" % __version__)
sys.exit(1)
# Ignore noisy twisted deprecation warnings
@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index f55ecc213..f74f89bda 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,18 +1,4 @@
-import sys
-from six.moves import copyreg
-
-if sys.version_info[0] == 2:
- from urlparse import urlparse
-
- # workaround for https://bugs.python.org/issue7904 - Python < 2.7
- if urlparse('s3://bucket/key').netloc != 'bucket':
- from urlparse import uses_netloc
- uses_netloc.append('s3')
-
- # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
- if urlparse('s3://bucket/key?key=value').query != 'key=value':
- from urlparse import uses_query
- uses_query.append('s3')
+import copyreg
# Undo what Twisted's perspective broker adds to pickle register
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index dc6b59fe0..ec78f7c91 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,5 +1,5 @@
-from __future__ import print_function
-import sys, os
+import sys
+import os
import optparse
import cProfile
import inspect
@@ -14,6 +14,7 @@ from scrapy.utils.project import inside_project, get_project_settings
from scrapy.utils.python import garbage_collect
from scrapy.settings.deprecated import check_deprecated_settings
+
def _iter_command_classes(module_name):
# TODO: add `name` attribute to commands and and merge this function with
# scrapy.utils.spider.iter_spider_classes
@@ -25,6 +26,7 @@ def _iter_command_classes(module_name):
not obj == ScrapyCommand:
yield obj
+
def _get_commands_from_module(module, inproject):
d = {}
for cmd in _iter_command_classes(module):
@@ -33,6 +35,7 @@ def _get_commands_from_module(module, inproject):
d[cmdname] = cmd()
return d
+
def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
cmds = {}
for entry_point in pkg_resources.iter_entry_points(group):
@@ -43,6 +46,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
raise Exception("Invalid entry point %s" % entry_point.name)
return cmds
+
def _get_commands_dict(settings, inproject):
cmds = _get_commands_from_module('scrapy.commands', inproject)
cmds.update(_get_commands_from_entry_points(inproject))
@@ -51,6 +55,7 @@ def _get_commands_dict(settings, inproject):
cmds.update(_get_commands_from_module(cmds_module, inproject))
return cmds
+
def _pop_command_name(argv):
i = 0
for arg in argv[1:]:
@@ -59,13 +64,15 @@ def _pop_command_name(argv):
return arg
i += 1
+
def _print_header(settings, inproject):
if inproject:
- print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
- settings['BOT_NAME']))
+ print("Scrapy %s - project: %s\n" % (scrapy.__version__,
+ settings['BOT_NAME']))
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
+
def _print_commands(settings, inproject):
_print_header(settings, inproject)
print("Usage:")
@@ -80,11 +87,13 @@ def _print_commands(settings, inproject):
print()
print('Use "scrapy <command> -h" to see more info about a command')
+
def _print_unknown_command(settings, cmdname, inproject):
_print_header(settings, inproject)
print("Unknown command: %s\n" % cmdname)
print('Use "scrapy" to see available commands')
+
def _run_print_help(parser, func, *a, **kw):
try:
func(*a, **kw)
@@ -95,41 +104,27 @@ def _run_print_help(parser, func, *a, **kw):
parser.print_help()
sys.exit(2)
+
def execute(argv=None, settings=None):
if argv is None:
argv = sys.argv
- # --- backwards compatibility for scrapy.conf.settings singleton ---
- if settings is None and 'scrapy.conf' in sys.modules:
- from scrapy import conf
- if hasattr(conf, 'settings'):
- settings = conf.settings
- # ------------------------------------------------------------------
-
if settings is None:
settings = get_project_settings()
# set EDITOR from environment if available
try:
editor = os.environ['EDITOR']
- except KeyError: pass
+ except KeyError:
+ pass
else:
settings['EDITOR'] = editor
check_deprecated_settings(settings)
- # --- backwards compatibility for scrapy.conf.settings singleton ---
- import warnings
- from scrapy.exceptions import ScrapyDeprecationWarning
- with warnings.catch_warnings():
- warnings.simplefilter("ignore", ScrapyDeprecationWarning)
- from scrapy import conf
- conf.settings = settings
- # ------------------------------------------------------------------
-
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
- parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
- conflict_handler='resolve')
+ parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
+ conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
sys.exit(0)
@@ -150,12 +145,14 @@ def execute(argv=None, settings=None):
_run_print_help(parser, _run_command, cmd, args, opts)
sys.exit(cmd.exitcode)
+
def _run_command(cmd, args, opts):
if opts.profile:
_run_command_profiled(cmd, args, opts)
else:
cmd.run(args, opts)
+
def _run_command_profiled(cmd, args, opts):
if opts.profile:
sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
@@ -165,6 +162,7 @@ def _run_command_profiled(cmd, args, opts):
if opts.profile:
p.dump_stats(opts.profile)
+
if __name__ == '__main__':
try:
execute()
diff --git a/scrapy/command.py b/scrapy/command.py
deleted file mode 100644
index 3e1219bbc..000000000
--- a/scrapy/command.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.command` is deprecated, "
- "use `scrapy.commands` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.commands import *
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 43b420821..0b24193c2 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -47,7 +47,7 @@ class ScrapyCommand(object):
def help(self):
"""An extensive help for the command. It will be shown when using the
- "help" command. It can contain newlines, since not post-formatting will
+ "help" command. It can contain newlines, since no post-formatting will
be applied to its contents.
"""
return self.long_desc()
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 90c8d56a2..7bbe362e7 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,8 +1,7 @@
import sys
import time
import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
import scrapy
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index b8a9ef989..9d4437a47 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,12 +1,10 @@
-from __future__ import print_function
import time
-import sys
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult
from scrapy.commands import ScrapyCommand
from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, set_environ
from scrapy.utils.conf import build_component_list
@@ -68,16 +66,17 @@ class Command(ScrapyCommand):
spider_loader = self.crawler_process.spider_loader
- for spidername in args or spider_loader.list():
- spidercls = spider_loader.load(spidername)
- spidercls.start_requests = lambda s: conman.from_spider(s, result)
+ with set_environ(SCRAPY_CHECK='true'):
+ for spidername in args or spider_loader.list():
+ spidercls = spider_loader.load(spidername)
+ spidercls.start_requests = lambda s: conman.from_spider(s, result)
- tested_methods = conman.tested_methods_from_spidercls(spidercls)
- if opts.list:
- for method in tested_methods:
- contract_reqs[spidercls.name].append(method)
- elif tested_methods:
- self.crawler_process.crawl(spidercls)
+ tested_methods = conman.tested_methods_from_spidercls(spidercls)
+ if opts.list:
+ for method in tested_methods:
+ contract_reqs[spidercls.name].append(method)
+ elif tested_methods:
+ self.crawler_process.crawl(spidercls)
# start checks
if opts.list:
@@ -95,4 +94,3 @@ class Command(ScrapyCommand):
result.printErrors()
result.printSummary(start, stop)
self.exitcode = int(not result.wasSuccessful())
-
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 4b986bf9d..8093fd402 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -56,3 +56,6 @@ class Command(ScrapyCommand):
self.crawler_process.crawl(spname, **opts.spargs)
self.crawler_process.start()
+
+ if self.crawler_process.bootstrap_failed:
+ self.exitcode = 1
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 7d4840529..0e149941d 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,5 +1,4 @@
-from __future__ import print_function
-import sys, six
+import sys
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
@@ -8,6 +7,7 @@ from scrapy.exceptions import UsageError
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+
class Command(ScrapyCommand):
requires_project = False
@@ -24,12 +24,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("--spider", dest="spider",
- help="use this spider")
- parser.add_option("--headers", dest="headers", action="store_true", \
- help="print response HTTP headers instead of body")
- parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
- default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+ parser.add_option("--spider", dest="spider", help="use this spider")
+ parser.add_option("--headers", dest="headers", action="store_true",
+ help="print response HTTP headers instead of body")
+ parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
+ default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():
@@ -45,8 +44,7 @@ class Command(ScrapyCommand):
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
- bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
- bytes_writer.write(bytes_ + b'\n')
+ sys.stdout.buffer.write(bytes_ + b'\n')
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index d5498bb5c..adb01fa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import os
import shutil
import string
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..54d7bb228 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,6 +1,6 @@
-from __future__ import print_function
from scrapy.commands import ScrapyCommand
+
class Command(ScrapyCommand):
requires_project = True
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 69418a478..ff6f1d8cd 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import json
import logging
@@ -51,18 +50,21 @@ class Command(ScrapyCommand):
help="use this callback for parsing, instead looking for a callback")
parser.add_option("-m", "--meta", dest="meta",
help="inject extra meta into the Request, it must be a valid raw json string")
+ parser.add_option("--cbkwargs", dest="cbkwargs",
+ help="inject extra callback kwargs into the Request, it must be a valid raw json string")
parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
help="maximum depth for parsing requests [default: %default]")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="print each depth level one by one")
-
@property
def max_level(self):
- levels = list(self.items.keys()) + list(self.requests.keys())
- if not levels:
- return 0
- return max(levels)
+ max_items, max_requests = 0, 0
+ if self.items:
+ max_items = max(self.items)
+ if self.requests:
+ max_requests = max(self.requests)
+ return max(max_items, max_requests)
def add_items(self, lvl, new_items):
old_items = self.items.get(lvl, [])
@@ -83,9 +85,8 @@ class Command(ScrapyCommand):
def print_requests(self, lvl=None, colour=True):
if lvl is None:
- levels = list(self.requests.keys())
- if levels:
- requests = self.requests[max(levels)]
+ if self.requests:
+ requests = self.requests[max(self.requests)]
else:
requests = []
else:
@@ -111,10 +112,11 @@ class Command(ScrapyCommand):
if not opts.nolinks:
self.print_requests(colour=colour)
- def run_callback(self, response, cb):
+ def run_callback(self, response, callback, cb_kwargs=None):
+ cb_kwargs = cb_kwargs or {}
items, requests = [], []
- for x in iterate_spider_output(cb(response)):
+ for x in iterate_spider_output(callback(response, **cb_kwargs)):
if isinstance(x, (BaseItem, dict)):
items.append(x)
elif isinstance(x, Request):
@@ -142,8 +144,7 @@ class Command(ScrapyCommand):
else:
self.spidercls = spidercls_for_request(spider_loader, Request(url))
if not self.spidercls:
- logger.error('Unable to find spider for: %(url)s',
- {'url': url})
+ logger.error('Unable to find spider for: %(url)s', {'url': url})
# Request requires callback argument as callable or None, not string
request = Request(url, None)
@@ -160,7 +161,7 @@ class Command(ScrapyCommand):
{'url': url})
def prepare_request(self, spider, request, opts):
- def callback(response):
+ def callback(response, **cb_kwargs):
# memorize first request
if not self.first_response:
self.first_response = response
@@ -175,7 +176,7 @@ class Command(ScrapyCommand):
if not cb:
logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
- {'url': response.url, 'spider': spider.name})
+ {'url': response.url, 'spider': spider.name})
return
else:
cb = 'parse'
@@ -192,7 +193,7 @@ class Command(ScrapyCommand):
# parse items and requests
depth = response.meta['_depth']
- items, requests = self.run_callback(response, cb)
+ items, requests = self.run_callback(response, cb, cb_kwargs)
if opts.pipelines:
itemproc = self.pcrawler.engine.scraper.itemproc
for item in items:
@@ -207,10 +208,14 @@ class Command(ScrapyCommand):
req.callback = callback
return requests
- #update request meta if any extra meta was passed through the --meta/-m opts.
+ # update request meta if any extra meta was passed through the --meta/-m opts.
if opts.meta:
request.meta.update(opts.meta)
+ # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
+ if opts.cbkwargs:
+ request.cb_kwargs.update(opts.cbkwargs)
+
request.meta['_depth'] = 1
request.meta['_callback'] = request.callback
request.callback = callback
@@ -221,23 +226,29 @@ class Command(ScrapyCommand):
self.process_spider_arguments(opts)
self.process_request_meta(opts)
+ self.process_request_cb_kwargs(opts)
def process_spider_arguments(self, opts):
-
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
def process_request_meta(self, opts):
-
if opts.meta:
try:
opts.meta = json.loads(opts.meta)
except ValueError:
- raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
- "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+ raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+ "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+ def process_request_cb_kwargs(self, opts):
+ if opts.cbkwargs:
+ try:
+ opts.cbkwargs = json.loads(opts.cbkwargs)
+ except ValueError:
+ raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
+ "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
def run(self, args, opts):
# parse arguments
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index a98033dd1..57d8471ca 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -60,14 +60,13 @@ class Command(ScrapyCommand):
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
- valid_output_formats = feed_exporters.keys()
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
- if opts.output_format not in valid_output_formats:
+ if opts.output_format not in feed_exporters:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
- tuple(valid_output_formats)))
+ tuple(feed_exporters)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):
@@ -87,3 +86,6 @@ class Command(ScrapyCommand):
self.crawler_process.crawl(spidercls, **opts.spargs)
self.crawler_process.start()
+
+ if self.crawler_process.bootstrap_failed:
+ self.exitcode = 1
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bee52f06a..603bafb9f 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,9 +1,9 @@
-from __future__ import print_function
import json
from scrapy.commands import ScrapyCommand
from scrapy.settings import BaseSettings
+
class Command(ScrapyCommand):
requires_project = False
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 40a58d94a..d44a32d5f 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -6,8 +6,8 @@ See documentation in docs/topics/shell.rst
from threading import Thread
from scrapy.commands import ScrapyCommand
-from scrapy.shell import Shell
from scrapy.http import Request
+from scrapy.shell import Shell
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
from scrapy.utils.url import guess_scheme
@@ -28,7 +28,8 @@ class Command(ScrapyCommand):
return "Interactive scraping console"
def long_desc(self):
- return "Interactive console for scraping the given url"
+ return ("Interactive console for scraping the given url or file. "
+ "Use ./file.html syntax or full path for local file.")
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index c17aaf442..b123e5c84 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import re
import os
import string
@@ -44,8 +43,8 @@ class Command(ScrapyCommand):
return False
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
- print('Error: Project names must begin with a letter and contain'\
- ' only\nletters, numbers and underscores')
+ print('Error: Project names must begin with a letter and contain'
+ ' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
else:
@@ -107,8 +106,8 @@ class Command(ScrapyCommand):
string.Template(path).substitute(project_name=project_name))
render_templatefile(tplfile, project_name=project_name,
ProjectName=string_camelcase(project_name))
- print("New Scrapy project %r, using template directory %r, created in:" % \
- (project_name, self.templates_dir))
+ print("New Scrapy project '%s', using template directory '%s', "
+ "created in:" % (project_name, self.templates_dir))
print(" %s\n" % abspath(project_dir))
print("You can start your first spider with:")
print(" cd %s" % project_dir)
@@ -119,4 +118,3 @@ class Command(ScrapyCommand):
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
join(scrapy.__path__[0], 'templates')
return join(_templates_base_dir, 'project')
-
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 577365c3b..1516c5997 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions
@@ -30,4 +28,3 @@ class Command(ScrapyCommand):
print(patt % (name, version))
else:
print("Scrapy %s" % scrapy.__version__)
-
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 59e665016..41e77ba3b 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,6 +1,7 @@
-from scrapy.commands import fetch, ScrapyCommand
+from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
+
class Command(fetch.Command):
def short_desc(self):
diff --git a/scrapy/conf.py b/scrapy/conf.py
deleted file mode 100644
index 23efc6ffd..000000000
--- a/scrapy/conf.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# This module is kept for backwards compatibility, so users can import
-# scrapy.conf.settings and get the settings they expect
-
-import sys
-
-if 'scrapy.cmdline' not in sys.modules:
- from scrapy.utils.project import get_project_settings
- settings = get_project_settings()
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.conf` is deprecated, use `crawler.settings` attribute instead",
- ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 5eaee3d11..7b6591d86 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -1,6 +1,7 @@
import sys
import re
from functools import wraps
+from inspect import getmembers
from unittest import TestCase
from scrapy.http import Request
@@ -17,7 +18,7 @@ class ContractsManager(object):
def tested_methods_from_spidercls(self, spidercls):
methods = []
- for key, value in vars(spidercls).items():
+ for key, value in getmembers(spidercls):
if (callable(value) and value.__doc__ and
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
methods.append(key)
@@ -41,23 +42,38 @@ class ContractsManager(object):
requests = []
for method in self.tested_methods_from_spidercls(type(spider)):
bound_method = spider.__getattribute__(method)
- requests.append(self.from_method(bound_method, results))
+ try:
+ requests.append(self.from_method(bound_method, results))
+ except Exception:
+ case = _create_testcase(bound_method, 'contract')
+ results.addError(case, sys.exc_info())
return requests
def from_method(self, method, results):
contracts = self.extract_contracts(method)
if contracts:
+ request_cls = Request
+ for contract in contracts:
+ if contract.request_cls is not None:
+ request_cls = contract.request_cls
+
# calculate request args
- args, kwargs = get_spec(Request.__init__)
+ args, kwargs = get_spec(request_cls.__init__)
+
+ # Don't filter requests to allow
+ # testing different callbacks on the same URL.
+ kwargs['dont_filter'] = True
kwargs['callback'] = method
+
for contract in contracts:
kwargs = contract.adjust_request_args(kwargs)
- # create and prepare request
args.remove('self')
+
+ # check if all positional arguments are defined in kwargs
if set(args).issubset(set(kwargs)):
- request = Request(**kwargs)
+ request = request_cls(**kwargs)
# execute pre and post hooks in order
for contract in reversed(contracts):
@@ -74,17 +90,17 @@ class ContractsManager(object):
cb = request.callback
@wraps(cb)
- def cb_wrapper(response):
+ def cb_wrapper(response, **cb_kwargs):
try:
- output = cb(response)
+ output = cb(response, **cb_kwargs)
output = list(iterate_spider_output(output))
- except:
+ except Exception:
case = _create_testcase(method, 'callback')
results.addError(case, sys.exc_info())
def eb_wrapper(failure):
case = _create_testcase(method, 'errback')
- exc_info = failure.value, failure.type, failure.getTracebackObject()
+ exc_info = failure.type, failure.value, failure.getTracebackObject()
results.addError(case, exc_info)
request.callback = cb_wrapper
@@ -93,6 +109,7 @@ class ContractsManager(object):
class Contract(object):
""" Abstract class for contracts """
+ request_cls = None
def __init__(self, method, *args):
self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
@@ -104,7 +121,7 @@ class Contract(object):
cb = request.callback
@wraps(cb)
- def wrapper(response):
+ def wrapper(response, **cb_kwargs):
try:
results.startTest(self.testcase_pre)
self.pre_process(response)
@@ -116,7 +133,7 @@ class Contract(object):
else:
results.addSuccess(self.testcase_pre)
finally:
- return list(iterate_spider_output(cb(response)))
+ return list(iterate_spider_output(cb(response, **cb_kwargs)))
request.callback = wrapper
@@ -127,8 +144,8 @@ class Contract(object):
cb = request.callback
@wraps(cb)
- def wrapper(response):
- output = list(iterate_spider_output(cb(response)))
+ def wrapper(response, **cb_kwargs):
+ output = list(iterate_spider_output(cb(response, **cb_kwargs)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 20582503d..3002fc702 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,8 +1,10 @@
+import json
+
from scrapy.item import BaseItem
from scrapy.http import Request
from scrapy.exceptions import ContractFail
-from . import Contract
+from scrapy.contracts import Contract
# contracts
@@ -18,6 +20,20 @@ class UrlContract(Contract):
return args
+class CallbackKeywordArgumentsContract(Contract):
+ """ Contract to set the keyword arguments for the request.
+ The value should be a JSON-encoded dictionary, e.g.:
+
+ @cb_kwargs {"arg1": "some value"}
+ """
+
+ name = 'cb_kwargs'
+
+ def adjust_request_args(self, args):
+ args['cb_kwargs'] = json.loads(' '.join(self.args))
+ return args
+
+
class ReturnsContract(Contract):
""" Contract to check the output of a callback
@@ -70,8 +86,8 @@ class ReturnsContract(Contract):
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
- raise ContractFail("Returned %s %s, expected %s" % \
- (occurrences, self.obj_name, expected))
+ raise ContractFail("Returned %s %s, expected %s" %
+ (occurrences, self.obj_name, expected))
class ScrapesContract(Contract):
@@ -84,6 +100,7 @@ class ScrapesContract(Contract):
def post_process(self, output):
for x in output:
if isinstance(x, (BaseItem, dict)):
- for arg in self.args:
- if not arg in x:
- raise ContractFail("'%s' field is missing" % arg)
+ missing = [arg for arg in self.args if arg not in x]
+ if missing:
+ raise ContractFail(
+ "Missing fields: %s" % ", ".join(missing))
diff --git a/scrapy/contrib/__init__.py b/scrapy/contrib/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py
deleted file mode 100644
index 9c52c418f..000000000
--- a/scrapy/contrib/closespider.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.closespider` is deprecated, "
- "use `scrapy.extensions.closespider` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.closespider import *
diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py
deleted file mode 100644
index 2f5354239..000000000
--- a/scrapy/contrib/corestats.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.corestats` is deprecated, "
- "use `scrapy.extensions.corestats` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.corestats import *
diff --git a/scrapy/contrib/debug.py b/scrapy/contrib/debug.py
deleted file mode 100644
index a38f059ce..000000000
--- a/scrapy/contrib/debug.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.debug` is deprecated, "
- "use `scrapy.extensions.debug` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.debug import *
diff --git a/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/contrib/downloadermiddleware/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py b/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
deleted file mode 100644
index 90ebc46b6..000000000
--- a/scrapy/contrib/downloadermiddleware/ajaxcrawl.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.ajaxcrawl` is deprecated, "
- "use `scrapy.downloadermiddlewares.ajaxcrawl` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.ajaxcrawl import *
diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py
deleted file mode 100644
index 1322c9083..000000000
--- a/scrapy/contrib/downloadermiddleware/chunked.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.chunked` is deprecated, "
- "use `scrapy.downloadermiddlewares.chunked` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.chunked import *
diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py
deleted file mode 100644
index bad970690..000000000
--- a/scrapy/contrib/downloadermiddleware/cookies.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.cookies` is deprecated, "
- "use `scrapy.downloadermiddlewares.cookies` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.cookies import *
diff --git a/scrapy/contrib/downloadermiddleware/decompression.py b/scrapy/contrib/downloadermiddleware/decompression.py
deleted file mode 100644
index a541aa61e..000000000
--- a/scrapy/contrib/downloadermiddleware/decompression.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.decompression` is deprecated, "
- "use `scrapy.downloadermiddlewares.decompression` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.decompression import *
diff --git a/scrapy/contrib/downloadermiddleware/defaultheaders.py b/scrapy/contrib/downloadermiddleware/defaultheaders.py
deleted file mode 100644
index cf023dc8f..000000000
--- a/scrapy/contrib/downloadermiddleware/defaultheaders.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.defaultheaders` is deprecated, "
- "use `scrapy.downloadermiddlewares.defaultheaders` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.defaultheaders import *
diff --git a/scrapy/contrib/downloadermiddleware/downloadtimeout.py b/scrapy/contrib/downloadermiddleware/downloadtimeout.py
deleted file mode 100644
index 84bd06acf..000000000
--- a/scrapy/contrib/downloadermiddleware/downloadtimeout.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.downloadtimeout` is deprecated, "
- "use `scrapy.downloadermiddlewares.downloadtimeout` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.downloadtimeout import *
diff --git a/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/contrib/downloadermiddleware/httpauth.py
deleted file mode 100644
index a37ffa0dc..000000000
--- a/scrapy/contrib/downloadermiddleware/httpauth.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpauth` is deprecated, "
- "use `scrapy.downloadermiddlewares.httpauth` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpauth import *
diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py
deleted file mode 100644
index f5f068204..000000000
--- a/scrapy/contrib/downloadermiddleware/httpcache.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcache` is deprecated, "
- "use `scrapy.downloadermiddlewares.httpcache` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpcache import *
diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py
deleted file mode 100644
index 8a52ec50b..000000000
--- a/scrapy/contrib/downloadermiddleware/httpcompression.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpcompression` is deprecated, "
- "use `scrapy.downloadermiddlewares.httpcompression` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpcompression import *
diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py
deleted file mode 100644
index d94d85076..000000000
--- a/scrapy/contrib/downloadermiddleware/httpproxy.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.httpproxy` is deprecated, "
- "use `scrapy.downloadermiddlewares.httpproxy` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.httpproxy import *
diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py
deleted file mode 100644
index 824eee8ae..000000000
--- a/scrapy/contrib/downloadermiddleware/redirect.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.redirect` is deprecated, "
- "use `scrapy.downloadermiddlewares.redirect` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.redirect import *
diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py
deleted file mode 100644
index aafe0f508..000000000
--- a/scrapy/contrib/downloadermiddleware/retry.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.retry` is deprecated, "
- "use `scrapy.downloadermiddlewares.retry` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.retry import *
diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py
deleted file mode 100644
index 408f760a0..000000000
--- a/scrapy/contrib/downloadermiddleware/robotstxt.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.robotstxt` is deprecated, "
- "use `scrapy.downloadermiddlewares.robotstxt` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.robotstxt import *
diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py
deleted file mode 100644
index fa84a8206..000000000
--- a/scrapy/contrib/downloadermiddleware/stats.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.stats` is deprecated, "
- "use `scrapy.downloadermiddlewares.stats` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.stats import *
diff --git a/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/contrib/downloadermiddleware/useragent.py
deleted file mode 100644
index 893d5241c..000000000
--- a/scrapy/contrib/downloadermiddleware/useragent.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.downloadermiddleware.useragent` is deprecated, "
- "use `scrapy.downloadermiddlewares.useragent` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.useragent import *
diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py
deleted file mode 100644
index 12adaaddd..000000000
--- a/scrapy/contrib/exporter/__init__.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.exporter` is deprecated, "
- "use `scrapy.exporters` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.exporters import *
-from scrapy.exporters import PythonItemExporter
diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py
deleted file mode 100644
index 19651998a..000000000
--- a/scrapy/contrib/feedexport.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.feedexport` is deprecated, "
- "use `scrapy.extensions.feedexport` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.feedexport import *
diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py
deleted file mode 100644
index 196372fcb..000000000
--- a/scrapy/contrib/httpcache.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.httpcache` is deprecated, "
- "use `scrapy.extensions.httpcache` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.httpcache import *
diff --git a/scrapy/contrib/linkextractors/__init__.py b/scrapy/contrib/linkextractors/__init__.py
deleted file mode 100644
index 976658df3..000000000
--- a/scrapy/contrib/linkextractors/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors` is deprecated, "
- "use `scrapy.linkextractors` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors import *
diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py
deleted file mode 100644
index ff03da98f..000000000
--- a/scrapy/contrib/linkextractors/htmlparser.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.htmlparser` is deprecated, "
- "use `scrapy.linkextractors.htmlparser` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.htmlparser import *
diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py
deleted file mode 100644
index fc2b7de3c..000000000
--- a/scrapy/contrib/linkextractors/lxmlhtml.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.lxmlhtml` is deprecated, "
- "use `scrapy.linkextractors.lxmlhtml` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.lxmlhtml import *
diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py
deleted file mode 100644
index 97bda29c1..000000000
--- a/scrapy/contrib/linkextractors/regex.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.regex` is deprecated, "
- "use `scrapy.linkextractors.regex` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.regex import *
diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py
deleted file mode 100644
index a5a598208..000000000
--- a/scrapy/contrib/linkextractors/sgml.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.linkextractors.sgml` is deprecated, "
- "use `scrapy.linkextractors.sgml` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors.sgml import *
diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py
deleted file mode 100644
index 2b9453e18..000000000
--- a/scrapy/contrib/loader/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.loader` is deprecated, "
- "use `scrapy.loader` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.loader import *
diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py
deleted file mode 100644
index a59b2b7b1..000000000
--- a/scrapy/contrib/loader/common.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.loader.common` is deprecated, "
- "use `scrapy.loader.common` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.loader.common import *
diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py
deleted file mode 100644
index da7e484a5..000000000
--- a/scrapy/contrib/loader/processor.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.loader.processor` is deprecated, "
- "use `scrapy.loader.processors` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.loader.processors import *
diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py
deleted file mode 100644
index 62bc9b860..000000000
--- a/scrapy/contrib/logstats.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.logstats` is deprecated, "
- "use `scrapy.extensions.logstats` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.logstats import *
diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py
deleted file mode 100644
index 4f6e4760e..000000000
--- a/scrapy/contrib/memdebug.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.memdebug` is deprecated, "
- "use `scrapy.extensions.memdebug` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.memdebug import *
diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py
deleted file mode 100644
index e13bd78f3..000000000
--- a/scrapy/contrib/memusage.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.memusage` is deprecated, "
- "use `scrapy.extensions.memusage` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.memusage import *
diff --git a/scrapy/contrib/pipeline/__init__.py b/scrapy/contrib/pipeline/__init__.py
deleted file mode 100644
index aedf34a3f..000000000
--- a/scrapy/contrib/pipeline/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline` is deprecated, "
- "use `scrapy.pipelines` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines import *
diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py
deleted file mode 100644
index cd1238b5d..000000000
--- a/scrapy/contrib/pipeline/files.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline.files` is deprecated, "
- "use `scrapy.pipelines.files` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines.files import *
diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py
deleted file mode 100644
index 4f5ce4c40..000000000
--- a/scrapy/contrib/pipeline/images.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline.images` is deprecated, "
- "use `scrapy.pipelines.images` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines.images import *
diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py
deleted file mode 100644
index 4b4fea560..000000000
--- a/scrapy/contrib/pipeline/media.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.pipeline.media` is deprecated, "
- "use `scrapy.pipelines.media` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.pipelines.media import *
diff --git a/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/contrib/spidermiddleware/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py
deleted file mode 100644
index 718803148..000000000
--- a/scrapy/contrib/spidermiddleware/depth.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.depth` is deprecated, "
- "use `scrapy.spidermiddlewares.depth` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.depth import *
diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py
deleted file mode 100644
index e39fb3f56..000000000
--- a/scrapy/contrib/spidermiddleware/httperror.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.httperror` is deprecated, "
- "use `scrapy.spidermiddlewares.httperror` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.httperror import *
diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py
deleted file mode 100644
index a5ed9ea7e..000000000
--- a/scrapy/contrib/spidermiddleware/offsite.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.offsite` is deprecated, "
- "use `scrapy.spidermiddlewares.offsite` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.offsite import *
diff --git a/scrapy/contrib/spidermiddleware/referer.py b/scrapy/contrib/spidermiddleware/referer.py
deleted file mode 100644
index fdf8d6659..000000000
--- a/scrapy/contrib/spidermiddleware/referer.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.referer` is deprecated, "
- "use `scrapy.spidermiddlewares.referer` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.referer import *
diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py
deleted file mode 100644
index 5e51add59..000000000
--- a/scrapy/contrib/spidermiddleware/urllength.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spidermiddleware.urllength` is deprecated, "
- "use `scrapy.spidermiddlewares.urllength` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spidermiddlewares.urllength import *
diff --git a/scrapy/contrib/spiders/__init__.py b/scrapy/contrib/spiders/__init__.py
deleted file mode 100644
index 56780533b..000000000
--- a/scrapy/contrib/spiders/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders` is deprecated, "
- "use `scrapy.spiders` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders import *
diff --git a/scrapy/contrib/spiders/crawl.py b/scrapy/contrib/spiders/crawl.py
deleted file mode 100644
index d20a8bb16..000000000
--- a/scrapy/contrib/spiders/crawl.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.crawl` is deprecated, "
- "use `scrapy.spiders.crawl` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.crawl import *
diff --git a/scrapy/contrib/spiders/feed.py b/scrapy/contrib/spiders/feed.py
deleted file mode 100644
index 5eea9a062..000000000
--- a/scrapy/contrib/spiders/feed.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.feed` is deprecated, "
- "use `scrapy.spiders.feed` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.feed import *
diff --git a/scrapy/contrib/spiders/init.py b/scrapy/contrib/spiders/init.py
deleted file mode 100644
index 6d1ec0aa9..000000000
--- a/scrapy/contrib/spiders/init.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.init` is deprecated, "
- "use `scrapy.spiders.init` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.init import *
diff --git a/scrapy/contrib/spiders/sitemap.py b/scrapy/contrib/spiders/sitemap.py
deleted file mode 100644
index 2ad231fd8..000000000
--- a/scrapy/contrib/spiders/sitemap.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiders.sitemap` is deprecated, "
- "use `scrapy.spiders.sitemap` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders.sitemap import *
diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py
deleted file mode 100644
index 06afc8bfc..000000000
--- a/scrapy/contrib/spiderstate.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.spiderstate` is deprecated, "
- "use `scrapy.extensions.spiderstate` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.spiderstate import *
diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py
deleted file mode 100644
index f9c9a37f5..000000000
--- a/scrapy/contrib/statsmailer.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.statsmailer` is deprecated, "
- "use `scrapy.extensions.statsmailer` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.statsmailer import *
diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py
deleted file mode 100644
index d5c234871..000000000
--- a/scrapy/contrib/throttle.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib.throttle` is deprecated, "
- "use `scrapy.extensions.throttle` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.throttle import *
diff --git a/scrapy/contrib_exp/__init__.py b/scrapy/contrib_exp/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib_exp/downloadermiddleware/__init__.py b/scrapy/contrib_exp/downloadermiddleware/__init__.py
deleted file mode 100644
index e69de29bb..000000000
diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py
deleted file mode 100644
index 1f8490587..000000000
--- a/scrapy/contrib_exp/downloadermiddleware/decompression.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib_exp.downloadermiddleware.decompression` is deprecated, "
- "use `scrapy.downloadermiddlewares.decompression` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.downloadermiddlewares.decompression import DecompressionMiddleware
diff --git a/scrapy/contrib_exp/iterators.py b/scrapy/contrib_exp/iterators.py
deleted file mode 100644
index c59f47bcc..000000000
--- a/scrapy/contrib_exp/iterators.py
+++ /dev/null
@@ -1,6 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.contrib_exp.iterators` is deprecated, use `scrapy.utils.iterators` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.utils.iterators import xmliter_lxml
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index d835e65f7..157dc3418 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,19 +1,16 @@
-from __future__ import absolute_import
import random
-import warnings
from time import time
from datetime import datetime
from collections import deque
-import six
from twisted.internet import reactor, defer, task
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
-from .middleware import DownloaderMiddlewareManager
-from .handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.core.downloader.handlers import DownloadHandlers
class Slot(object):
@@ -60,10 +57,6 @@ class Slot(object):
def _get_concurrency_delay(concurrency, spider, settings):
delay = settings.getfloat('DOWNLOAD_DELAY')
- if hasattr(spider, 'DOWNLOAD_DELAY'):
- warnings.warn("%s.DOWNLOAD_DELAY attribute is deprecated, use %s.download_delay instead" %
- (type(spider).__name__, type(spider).__name__))
- delay = spider.DOWNLOAD_DELAY
if hasattr(spider, 'download_delay'):
delay = spider.download_delay
@@ -75,6 +68,8 @@ def _get_concurrency_delay(concurrency, spider, settings):
class Downloader(object):
+ DOWNLOAD_SLOT = 'download_slot'
+
def __init__(self, crawler):
self.settings = crawler.settings
self.signals = crawler.signals
@@ -111,8 +106,8 @@ class Downloader(object):
return key, self.slots[key]
def _get_slot_key(self, request, spider):
- if 'download_slot' in request.meta:
- return request.meta['download_slot']
+ if self.DOWNLOAD_SLOT in request.meta:
+ return request.meta[self.DOWNLOAD_SLOT]
key = urlparse_cached(request).hostname or ''
if self.ip_concurrency:
@@ -122,13 +117,16 @@ class Downloader(object):
def _enqueue_request(self, request, spider):
key, slot = self._get_slot(request, spider)
- request.meta['download_slot'] = key
+ request.meta[self.DOWNLOAD_SLOT] = key
def _deactivate(response):
slot.active.remove(request)
return response
slot.active.add(request)
+ self.signals.send_catch_log(signal=signals.request_reached_downloader,
+ request=request,
+ spider=spider)
deferred = defer.Deferred().addBoth(_deactivate)
slot.queue.append((request, deferred))
self._process_queue(spider, slot)
@@ -189,7 +187,7 @@ class Downloader(object):
def close(self):
self._slot_gc_loop.stop()
- for slot in six.itervalues(self.slots):
+ for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 783d4c383..6e023ebcc 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,105 +1,93 @@
from OpenSSL import SSL
-from twisted.internet.ssl import ClientContextFactory
+from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust, AcceptableCiphers
+from twisted.web.client import BrowserLikePolicyForHTTPS
+from twisted.web.iweb import IPolicyForHTTPS
+from zope.interface.declarations import implementer
-from scrapy import twisted_version
-
-if twisted_version >= (14, 0, 0):
-
- from zope.interface.declarations import implementer
-
- from twisted.internet.ssl import (optionsForClientTLS,
- CertificateOptions,
- platformTrust)
- from twisted.web.client import BrowserLikePolicyForHTTPS
- from twisted.web.iweb import IPolicyForHTTPS
-
- from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
+from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
- @implementer(IPolicyForHTTPS)
- class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
- """
- Non-peer-certificate verifying HTTPS context factory
+@implementer(IPolicyForHTTPS)
+class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
+ """
+ Non-peer-certificate verifying HTTPS context factory
- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
- which allows TLS protocol negotiation
+ Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+ which allows TLS protocol negotiation
- 'A TLS/SSL connection established with [this method] may
- understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
- """
+ 'A TLS/SSL connection established with [this method] may
+ understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
+ """
- def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
- super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
- self._ssl_method = method
+ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
+ super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
+ self._ssl_method = method
+ self.tls_verbose_logging = tls_verbose_logging
+ if tls_ciphers:
+ self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
+ else:
+ self.tls_ciphers = DEFAULT_CIPHERS
- def getCertificateOptions(self):
- # setting verify=True will require you to provide CAs
- # to verify against; in other words: it's not that simple
+ @classmethod
+ def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
+ tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
+ tls_ciphers = settings['DOWNLOADER_CLIENT_TLS_CIPHERS']
+ return cls(method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, *args, **kwargs)
- # backward-compatible SSL/TLS method:
- #
- # * this will respect `method` attribute in often recommended
- # `ScrapyClientContextFactory` subclass
- # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
- #
- # * getattr() for `_ssl_method` attribute for context factories
- # not calling super(..., self).__init__
- return CertificateOptions(verify=False,
- method=getattr(self, 'method',
- getattr(self, '_ssl_method', None)),
- fixBrokenPeers=True,
- acceptableCiphers=DEFAULT_CIPHERS)
+ def getCertificateOptions(self):
+ # setting verify=True will require you to provide CAs
+ # to verify against; in other words: it's not that simple
- # kept for old-style HTTP/1.0 downloader context twisted calls,
- # e.g. connectSSL()
- def getContext(self, hostname=None, port=None):
- return self.getCertificateOptions().getContext()
+ # backward-compatible SSL/TLS method:
+ #
+ # * this will respect `method` attribute in often recommended
+ # `ScrapyClientContextFactory` subclass
+ # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
+ #
+ # * getattr() for `_ssl_method` attribute for context factories
+ # not calling super(..., self).__init__
+ return CertificateOptions(verify=False,
+ method=getattr(self, 'method',
+ getattr(self, '_ssl_method', None)),
+ fixBrokenPeers=True,
+ acceptableCiphers=self.tls_ciphers)
- def creatorForNetloc(self, hostname, port):
- return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
+ # kept for old-style HTTP/1.0 downloader context twisted calls,
+ # e.g. connectSSL()
+ def getContext(self, hostname=None, port=None):
+ return self.getCertificateOptions().getContext()
+
+ def creatorForNetloc(self, hostname, port):
+ return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
+ verbose_logging=self.tls_verbose_logging)
- @implementer(IPolicyForHTTPS)
- class BrowserLikeContextFactory(ScrapyClientContextFactory):
- """
- Twisted-recommended context factory for web clients.
+@implementer(IPolicyForHTTPS)
+class BrowserLikeContextFactory(ScrapyClientContextFactory):
+ """
+ Twisted-recommended context factory for web clients.
- Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
- "The default is to use a BrowserLikePolicyForHTTPS,
- so unless you have special requirements you can leave this as-is."
+ Quoting the documentation of the :class:`~twisted.web.client.Agent` class:
- creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
- except this context factory allows setting the TLS/SSL method to use.
+ The default is to use a
+ :class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you
+ have special requirements you can leave this as-is.
- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
- which allows TLS protocol negotiation.
- """
- def creatorForNetloc(self, hostname, port):
+ :meth:`creatorForNetloc` is the same as
+ :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
+ factory allows setting the TLS/SSL method to use.
- # trustRoot set to platformTrust() will use the platform's root CAs.
- #
- # This means that a website like https://www.cacert.org will be rejected
- # by default, since CAcert.org CA certificate is seldom shipped.
- return optionsForClientTLS(hostname.decode("ascii"),
- trustRoot=platformTrust(),
- extraCertificateOptions={
- 'method': self._ssl_method,
- })
+ The default OpenSSL method is ``TLS_METHOD`` (also called
+ ``SSLv23_METHOD``) which allows TLS protocol negotiation.
+ """
+ def creatorForNetloc(self, hostname, port):
-else:
-
- class ScrapyClientContextFactory(ClientContextFactory):
- "A SSL context factory which is more permissive against SSL bugs."
- # see https://github.com/scrapy/scrapy/issues/82
- # and https://github.com/scrapy/scrapy/issues/26
- # and https://github.com/scrapy/scrapy/issues/981
-
- def __init__(self, method=SSL.SSLv23_METHOD):
- self.method = method
-
- def getContext(self, hostname=None, port=None):
- ctx = ClientContextFactory.getContext(self)
- # Enable all workarounds to SSL bugs as documented by
- # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html
- ctx.set_options(SSL.OP_ALL)
- return ctx
+ # trustRoot set to platformTrust() will use the platform's root CAs.
+ #
+ # This means that a website like https://www.cacert.org will be rejected
+ # by default, since CAcert.org CA certificate is seldom shipped.
+ return optionsForClientTLS(hostname.decode("ascii"),
+ trustRoot=platformTrust(),
+ extraCertificateOptions={
+ 'method': self._ssl_method,
+ })
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index bc5cd742e..e86680978 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,19 +1,20 @@
"""Download handlers for different schemes"""
import logging
+
from twisted.internet import defer
-import six
-from scrapy.exceptions import NotSupported, NotConfigured
-from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.misc import load_object
-from scrapy.utils.python import without_none_values
+
from scrapy import signals
+from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import create_instance, load_object
+from scrapy.utils.python import without_none_values
logger = logging.getLogger(__name__)
-class DownloadHandlers(object):
+class DownloadHandlers:
def __init__(self, crawler):
self._crawler = crawler
@@ -22,8 +23,9 @@ class DownloadHandlers(object):
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
- for scheme, clspath in six.iteritems(handlers):
+ for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
+ self._load_handler(scheme, skip_lazy=True)
crawler.signals.connect(self._close, signals.engine_stopped)
@@ -39,22 +41,31 @@ class DownloadHandlers(object):
self._notconfigured[scheme] = 'no handler available for that scheme'
return None
+ return self._load_handler(scheme)
+
+ def _load_handler(self, scheme, skip_lazy=False):
path = self._schemes[scheme]
try:
dhcls = load_object(path)
- dh = dhcls(self._crawler.settings)
+ if skip_lazy and getattr(dhcls, 'lazy', True):
+ return None
+ dh = create_instance(
+ objcls=dhcls,
+ settings=self._crawler.settings,
+ crawler=self._crawler,
+ )
except NotConfigured as ex:
self._notconfigured[scheme] = str(ex)
return None
except Exception as ex:
logger.error('Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
- exc_info=True, extra={'crawler': self._crawler})
+ exc_info=True, extra={'crawler': self._crawler})
self._notconfigured[scheme] = str(ex)
return None
else:
self._handlers[scheme] = dh
- return self._handlers[scheme]
+ return dh
def download_request(self, request, spider):
scheme = urlparse_cached(request).scheme
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index d102f2b73..a45b4ff3c 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -5,9 +5,8 @@ from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
-class DataURIDownloadHandler(object):
- def __init__(self, settings):
- super(DataURIDownloadHandler, self).__init__()
+class DataURIDownloadHandler:
+ lazy = False
@defers
def download_request(self, request, spider):
@@ -15,8 +14,8 @@ class DataURIDownloadHandler(object):
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
- if (issubclass(respcls, TextResponse) and
- uri.media_type.split('/')[0] == 'text'):
+ if (issubclass(respcls, TextResponse)
+ and uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 9346ce08d..0d94e3df0 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -1,11 +1,11 @@
from w3lib.url import file_uri_to_path
+
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
-class FileDownloadHandler(object):
- def __init__(self, settings):
- pass
+class FileDownloadHandler:
+ lazy = False
@defers
def download_request(self, request, spider):
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 933bc7e8d..1681c6df8 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,11 +30,11 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
from twisted.internet import reactor
-from twisted.protocols.ftp import FTPClient, CommandFailed
-from twisted.internet.protocol import Protocol, ClientCreator
+from twisted.internet.protocol import ClientCreator, Protocol
+from twisted.protocols.ftp import CommandFailed, FTPClient
from scrapy.http import Response
from scrapy.responsetypes import responsetypes
@@ -59,8 +59,12 @@ class ReceivedDataProtocol(Protocol):
def close(self):
self.body.close() if self.filename else self.body.seek(0)
-_CODE_RE = re.compile("\d+")
-class FTPDownloadHandler(object):
+
+_CODE_RE = re.compile(r"\d+")
+
+
+class FTPDownloadHandler:
+ lazy = False
CODE_MAPPING = {
"550": 404,
@@ -72,6 +76,10 @@ class FTPDownloadHandler(object):
self.default_password = settings['FTP_PASSWORD']
self.passive_mode = settings['FTP_PASSIVE_MODE']
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler.settings)
+
def download_request(self, request, spider):
parsed_url = urlparse_cached(request)
user = request.meta.get("ftp_user", self.default_user)
@@ -109,4 +117,3 @@ class FTPDownloadHandler(object):
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
return Response(url=request.url, status=httpcode, body=to_bytes(message))
raise result.type(result.value)
-
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index e4a7d8564..52535bd8b 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,15 +1,4 @@
-from __future__ import absolute_import
-from .http10 import HTTP10DownloadHandler
-from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
-
-
-# backwards compatibility
-class HttpDownloadHandler(HTTP10DownloadHandler):
-
- def __init__(self, *args, **kwargs):
- import warnings
- from scrapy.exceptions import ScrapyDeprecationWarning
- warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader'
- '.handlers.http10.HTTP10DownloadHandler instead',
- category=ScrapyDeprecationWarning, stacklevel=1)
- super(HttpDownloadHandler, self).__init__(*args, **kwargs)
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import (
+ HTTP11DownloadHandler as HTTPDownloadHandler,
+)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 0322bbe49..d4aa51bd1 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,15 +1,23 @@
"""Download handlers for http and https schemes
"""
from twisted.internet import reactor
-from scrapy.utils.misc import load_object
+
+from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_unicode
-class HTTP10DownloadHandler(object):
+class HTTP10DownloadHandler:
+ lazy = False
- def __init__(self, settings):
+ def __init__(self, settings, crawler=None):
self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+ self._settings = settings
+ self._crawler = crawler
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler.settings, crawler)
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
@@ -20,7 +28,11 @@ class HTTP10DownloadHandler(object):
def _connect(self, factory):
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
- return reactor.connectSSL(host, port, factory,
- self.ClientContextFactory())
+ client_context_factory = create_instance(
+ objcls=self.ClientContextFactory,
+ settings=self._settings,
+ crawler=self._crawler,
+ )
+ return reactor.connectSSL(host, port, factory, client_context_factory)
else:
return reactor.connectTCP(host, port, factory)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 038db7b47..5a5f6cf0a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -1,40 +1,37 @@
"""Download handlers for http and https schemes"""
-import re
import logging
+import re
+import warnings
from io import BytesIO
from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
-from zope.interface import implementer
-from twisted.internet import defer, reactor, protocol
+from twisted.internet import defer, protocol, reactor
+from twisted.internet.endpoints import TCP4ClientEndpoint
+from twisted.internet.error import TimeoutError
+from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
+from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH
-from twisted.internet.error import TimeoutError
-from twisted.web.http import _DataLoss, PotentialDataLoss
-from twisted.web.client import Agent, ProxyAgent, ResponseDone, \
- HTTPConnectionPool, ResponseFailed
-try:
- from twisted.web.client import URI
-except ImportError:
- from twisted.web.client import _URI as URI
-from twisted.internet.endpoints import TCP4ClientEndpoint
+from zope.interface import implementer
+from scrapy.core.downloader.tls import openssl_methods
+from scrapy.core.downloader.webclient import _parse
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
-from scrapy.core.downloader.webclient import _parse
-from scrapy.core.downloader.tls import openssl_methods
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_bytes, to_unicode
-from scrapy import twisted_version
+
logger = logging.getLogger(__name__)
-class HTTP11DownloadHandler(object):
+class HTTP11DownloadHandler:
+ lazy = False
- def __init__(self, settings):
+ def __init__(self, settings, crawler=None):
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
@@ -43,14 +40,23 @@ class HTTP11DownloadHandler(object):
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
- self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
+ self._contextFactory = create_instance(
+ objcls=self._contextFactoryClass,
+ settings=settings,
+ crawler=crawler,
+ method=self._sslMethod,
+ )
except TypeError:
# use context factory defaults
- self._contextFactory = self._contextFactoryClass()
+ self._contextFactory = create_instance(
+ objcls=self._contextFactoryClass,
+ settings=settings,
+ crawler=crawler,
+ )
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD).\
- Please upgrade your context factory class to handle it or ignore it.""" % (
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument and/or `tls_ciphers` argument.\
+ Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
@@ -58,12 +64,19 @@ class HTTP11DownloadHandler(object):
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
self._disconnect_timeout = 1
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler.settings, crawler)
+
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
- agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool,
+ agent = ScrapyAgent(
+ contextFactory=self._contextFactory,
+ pool=self._pool,
maxsize=getattr(spider, 'download_maxsize', self._default_maxsize),
warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),
- fail_on_dataloss=self._fail_on_dataloss)
+ fail_on_dataloss=self._fail_on_dataloss,
+ )
return agent.download_request(request)
def close(self):
@@ -100,13 +113,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
for it.
"""
- _responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
+ _responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
- def __init__(self, reactor, host, port, proxyConf, contextFactory,
- timeout=30, bindAddress=None):
+ def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
- super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost,
- proxyPort, timeout, bindAddress)
+ super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
@@ -115,8 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def requestTunnel(self, protocol):
"""Asks the proxy to open a tunnel."""
- tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort,
- self._proxyAuthHeader)
+ tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader)
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
protocol.dataReceived = self.processProxyResponse
@@ -139,16 +149,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
self._protocol.dataReceived = self._protocolDataReceived
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
if respm and int(respm.group('status')) == 200:
- try:
- # this sets proper Server Name Indication extension
- # but is only available for Twisted>=14.0
- sslOptions = self._contextFactory.creatorForNetloc(
- self._tunneledHost, self._tunneledPort)
- except AttributeError:
- # fall back to non-SNI SSL context factory
- sslOptions = self._contextFactory
- self._protocol.transport.startTLS(sslOptions,
- self._protocolFactory)
+ # set proper Server Name Indication extension
+ sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort)
+ self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
if respm:
@@ -166,8 +169,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def connect(self, protocolFactory):
self._protocolFactory = protocolFactory
- connectDeferred = super(TunnelingTCP4ClientEndpoint,
- self).connect(protocolFactory)
+ connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
connectDeferred.addErrback(self.connectFailed)
return self._tunnelReadyDeferred
@@ -177,7 +179,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
r"""
Return binary content of a CONNECT request.
- >>> from scrapy.utils.python import to_native_str as s
+ >>> from scrapy.utils.python import to_unicode as s
>>> s(tunnel_request_data("example.com", 8080))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(tunnel_request_data("example.com", 8080, b"123"))
@@ -204,42 +206,46 @@ class TunnelingAgent(Agent):
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
- super(TunnelingAgent, self).__init__(reactor, contextFactory,
- connectTimeout, bindAddress, pool)
+ super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
- if twisted_version >= (15, 0, 0):
- def _getEndpoint(self, uri):
- return TunnelingTCP4ClientEndpoint(
- self._reactor, uri.host, uri.port, self._proxyConf,
- self._contextFactory, self._endpointFactory._connectTimeout,
- self._endpointFactory._bindAddress)
- else:
- def _getEndpoint(self, scheme, host, port):
- return TunnelingTCP4ClientEndpoint(
- self._reactor, host, port, self._proxyConf,
- self._contextFactory, self._connectTimeout,
- self._bindAddress)
+ def _getEndpoint(self, uri):
+ return TunnelingTCP4ClientEndpoint(
+ reactor=self._reactor,
+ host=uri.host,
+ port=uri.port,
+ proxyConf=self._proxyConf,
+ contextFactory=self._contextFactory,
+ timeout=self._endpointFactory._connectTimeout,
+ bindAddress=self._endpointFactory._bindAddress,
+ )
- def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
- headers, bodyProducer, requestPath):
+ def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath):
# proxy host and port are required for HTTP pool `key`
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
- return super(TunnelingAgent, self)._requestWithEndpoint(key, endpoint, method, parsedURI,
- headers, bodyProducer, requestPath)
+ return super(TunnelingAgent, self)._requestWithEndpoint(
+ key=key,
+ endpoint=endpoint,
+ method=method,
+ parsedURI=parsedURI,
+ headers=headers,
+ bodyProducer=bodyProducer,
+ requestPath=requestPath,
+ )
class ScrapyProxyAgent(Agent):
- def __init__(self, reactor, proxyURI,
- connectTimeout=None, bindAddress=None, pool=None):
- super(ScrapyProxyAgent, self).__init__(reactor,
- connectTimeout=connectTimeout,
- bindAddress=bindAddress,
- pool=pool)
+ def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
+ super(ScrapyProxyAgent, self).__init__(
+ reactor=reactor,
+ connectTimeout=connectTimeout,
+ bindAddress=bindAddress,
+ pool=pool,
+ )
self._proxyURI = URI.fromBytes(proxyURI)
def request(self, method, uri, headers=None, bodyProducer=None):
@@ -248,16 +254,15 @@ class ScrapyProxyAgent(Agent):
"""
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
- if twisted_version >= (15, 0, 0):
- proxyEndpoint = self._getEndpoint(self._proxyURI)
- else:
- proxyEndpoint = self._getEndpoint(self._proxyURI.scheme,
- self._proxyURI.host,
- self._proxyURI.port)
- key = ("http-proxy", self._proxyURI.host, self._proxyURI.port)
- return self._requestWithEndpoint(key, proxyEndpoint, method,
- URI.fromBytes(uri), headers,
- bodyProducer, uri)
+ return self._requestWithEndpoint(
+ key=("http-proxy", self._proxyURI.host, self._proxyURI.port),
+ endpoint=self._getEndpoint(self._proxyURI),
+ method=method,
+ parsedURI=URI.fromBytes(uri),
+ headers=headers,
+ bodyProducer=bodyProducer,
+ requestPath=uri,
+ )
class ScrapyAgent(object):
@@ -285,18 +290,39 @@ class ScrapyAgent(object):
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
- if scheme == b'https' and not omitConnectTunnel:
- proxyConf = (proxyHost, proxyPort,
- request.headers.get(b'Proxy-Authorization', None))
- return self._TunnelingAgent(reactor, proxyConf,
- contextFactory=self._contextFactory, connectTimeout=timeout,
- bindAddress=bindaddress, pool=self._pool)
+ if omitConnectTunnel:
+ warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
+ "If you use Crawlera, it doesn't require this mode anymore, "
+ "so you should update scrapy-crawlera to 1.3.0+ "
+ "and remove '?noconnect' from the Crawlera URL.",
+ ScrapyDeprecationWarning)
+ if scheme == b'https' and not omitConnectTunnel:
+ proxyAuth = request.headers.get(b'Proxy-Authorization', None)
+ proxyConf = (proxyHost, proxyPort, proxyAuth)
+ return self._TunnelingAgent(
+ reactor=reactor,
+ proxyConf=proxyConf,
+ contextFactory=self._contextFactory,
+ connectTimeout=timeout,
+ bindAddress=bindaddress,
+ pool=self._pool,
+ )
else:
- return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'),
- connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
+ return self._ProxyAgent(
+ reactor=reactor,
+ proxyURI=to_bytes(proxy, encoding='ascii'),
+ connectTimeout=timeout,
+ bindAddress=bindaddress,
+ pool=self._pool,
+ )
- return self._Agent(reactor, contextFactory=self._contextFactory,
- connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
+ return self._Agent(
+ reactor=reactor,
+ contextFactory=self._contextFactory,
+ connectTimeout=timeout,
+ bindAddress=bindaddress,
+ pool=self._pool,
+ )
def download_request(self, request):
timeout = request.meta.get('download_timeout') or self._connectTimeout
@@ -327,8 +353,7 @@ class ScrapyAgent(object):
else:
bodyproducer = None
start_time = time()
- d = agent.request(
- method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
+ d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
@@ -383,8 +408,9 @@ class ScrapyAgent(object):
txresponse._transport._producer.abortConnection()
d = defer.Deferred(_cancel)
- txresponse.deliverBody(_ResponseReader(
- d, txresponse, request, maxsize, warnsize, fail_on_dataloss))
+ txresponse.deliverBody(
+ _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss)
+ )
# save response for timeouts
self._txresponse = txresponse
@@ -419,22 +445,20 @@ class _RequestBodyProducer(object):
class _ResponseReader(protocol.Protocol):
- def __init__(self, finished, txresponse, request, maxsize, warnsize,
- fail_on_dataloss):
+ def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss):
self._finished = finished
self._txresponse = txresponse
self._request = request
self._bodybuf = BytesIO()
- self._maxsize = maxsize
- self._warnsize = warnsize
+ self._maxsize = maxsize
+ self._warnsize = warnsize
self._fail_on_dataloss = fail_on_dataloss
self._fail_on_dataloss_warned = False
self._reached_warnsize = False
self._bytes_received = 0
def dataReceived(self, bodyBytes):
- # This maybe called several times after cancel was called with buffered
- # data.
+ # This maybe called several times after cancel was called with buffered data.
if self._finished.called:
return
@@ -447,8 +471,7 @@ class _ResponseReader(protocol.Protocol):
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
- # Clear buffer earlier to avoid keeping data in memory for a long
- # time.
+ # Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()
@@ -478,10 +501,10 @@ class _ResponseReader(protocol.Protocol):
return
elif not self._fail_on_dataloss_warned:
- logger.warn("Got data loss in %s. If you want to process broken "
- "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
- " -- This message won't be shown in further requests",
- self._txresponse.request.absoluteURI.decode())
+ logger.warning("Got data loss in %s. If you want to process broken "
+ "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
+ " -- This message won't be shown in further requests",
+ self._txresponse.request.absoluteURI.decode())
self._fail_on_dataloss_warned = True
self._finished.errback(reason)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d8bbdd326..40a1fa48e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,9 +1,10 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.exceptions import NotConfigured
-from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.boto import is_botocore
-from .http import HTTPDownloadHandler
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import create_instance
def _get_boto_connection():
@@ -21,7 +22,7 @@ def _get_boto_connection():
return http_request.headers
try:
- import boto.auth
+ import boto.auth # noqa: F401
except ImportError:
_S3Connection = _v19_S3Connection
else:
@@ -30,11 +31,12 @@ def _get_boto_connection():
return _S3Connection
-class S3DownloadHandler(object):
-
- def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
- httpdownloadhandler=HTTPDownloadHandler, **kw):
+class S3DownloadHandler:
+ def __init__(self, settings, *,
+ crawler=None,
+ aws_access_key_id=None, aws_secret_access_key=None,
+ httpdownloadhandler=HTTPDownloadHandler, **kw):
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
@@ -67,7 +69,16 @@ class S3DownloadHandler(object):
except Exception as ex:
raise NotConfigured(str(ex))
- self._download_http = httpdownloadhandler(settings).download_request
+ _http_handler = create_instance(
+ objcls=httpdownloadhandler,
+ settings=settings,
+ crawler=crawler,
+ )
+ self._download_http = _http_handler.download_request
+
+ @classmethod
+ def from_crawler(cls, crawler, **kwargs):
+ return cls(crawler.settings, crawler=crawler, **kwargs)
def download_request(self, request, spider):
p = urlparse_cached(request)
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index c3b23e284..9c0014206 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,13 +3,12 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
-import six
-
from twisted.internet import defer
+from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
-from scrapy.utils.defer import mustbe_deferred
+from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
from scrapy.utils.conf import build_component_list
@@ -26,21 +25,21 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_request'):
self.methods['process_request'].append(mw.process_request)
if hasattr(mw, 'process_response'):
- self.methods['process_response'].insert(0, mw.process_response)
+ self.methods['process_response'].appendleft(mw.process_response)
if hasattr(mw, 'process_exception'):
- self.methods['process_exception'].insert(0, mw.process_exception)
+ self.methods['process_exception'].appendleft(mw.process_exception)
def download(self, download_func, request, spider):
@defer.inlineCallbacks
def process_request(request):
for method in self.methods['process_request']:
- response = yield method(request=request, spider=spider)
- assert response is None or isinstance(response, (Response, Request)), \
- 'Middleware %s.process_request must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
+ response = yield deferred_from_coro(method(request=request, spider=spider))
+ if response is not None and not isinstance(response, (Response, Request)):
+ raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
+ (method.__self__.__class__.__name__, response.__class__.__name__))
if response:
defer.returnValue(response)
- defer.returnValue((yield download_func(request=request,spider=spider)))
+ defer.returnValue((yield download_func(request=request, spider=spider)))
@defer.inlineCallbacks
def process_response(response):
@@ -49,11 +48,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
defer.returnValue(response)
for method in self.methods['process_response']:
- response = yield method(request=request, response=response,
- spider=spider)
- assert isinstance(response, (Response, Request)), \
- 'Middleware %s.process_response must return Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response))
+ response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
+ if not isinstance(response, (Response, Request)):
+ raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
+ (method.__self__.__class__.__name__, type(response)))
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@@ -62,11 +60,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
def process_exception(_failure):
exception = _failure.value
for method in self.methods['process_exception']:
- response = yield method(request=request, exception=exception,
- spider=spider)
- assert response is None or isinstance(response, (Response, Request)), \
- 'Middleware %s.process_exception must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response))
+ response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
+ if response is not None and not isinstance(response, (Response, Request)):
+ raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
+ (method.__self__.__class__.__name__, type(response)))
if response:
defer.returnValue(response)
defer.returnValue(_failure)
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index df8051182..4ed482058 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,17 +1,24 @@
import logging
+
from OpenSSL import SSL
+from service_identity.exceptions import CertificateError
+from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
+from twisted.internet.ssl import AcceptableCiphers
from scrapy import twisted_version
+from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
logger = logging.getLogger(__name__)
+
METHOD_SSLv3 = 'SSLv3'
METHOD_TLS = 'TLS'
METHOD_TLSv10 = 'TLSv1.0'
METHOD_TLSv11 = 'TLSv1.1'
METHOD_TLSv12 = 'TLSv1.2'
+
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
@@ -20,69 +27,66 @@ openssl_methods = {
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
}
-if twisted_version >= (14, 0, 0):
- # ClientTLSOptions requires a recent-enough version of Twisted.
- # Not having ScrapyClientTLSOptions should not matter for older
- # Twisted versions because it is not used in the fallback
- # ScrapyClientContextFactory.
- # taken from twisted/twisted/internet/_sslverify.py
-
- try:
- # XXX: this try-except is not needed in Twisted 17.0.0+ because
- # it requires pyOpenSSL 0.16+.
- from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
- except ImportError:
- SSL_CB_HANDSHAKE_START = 0x10
- SSL_CB_HANDSHAKE_DONE = 0x20
-
- from twisted.internet.ssl import AcceptableCiphers
- from twisted.internet._sslverify import (ClientTLSOptions,
- verifyHostname,
- VerificationError)
- try:
- # XXX: this import would fail on Debian jessie with system installed
- # service_identity library, due to lack of cryptography.x509 dependency
- # See https://github.com/pyca/service_identity/issues/21
- from service_identity.exceptions import CertificateError
- verification_errors = (CertificateError, VerificationError)
- except ImportError:
- verification_errors = VerificationError
-
- if twisted_version < (17, 0, 0):
- from twisted.internet._sslverify import _maybeSetHostNameIndication
- set_tlsext_host_name = _maybeSetHostNameIndication
- else:
- def set_tlsext_host_name(connection, hostNameBytes):
- connection.set_tlsext_host_name(hostNameBytes)
+if twisted_version < (17, 0, 0):
+ from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
+else:
+ def set_tlsext_host_name(connection, hostNameBytes):
+ connection.set_tlsext_host_name(hostNameBytes)
- class ScrapyClientTLSOptions(ClientTLSOptions):
- """
- SSL Client connection creator ignoring certificate verification errors
- (for genuinely invalid certificates or bugs in verification code).
+class ScrapyClientTLSOptions(ClientTLSOptions):
+ """
+ SSL Client connection creator ignoring certificate verification errors
+ (for genuinely invalid certificates or bugs in verification code).
- Same as Twisted's private _sslverify.ClientTLSOptions,
- except that VerificationError, CertificateError and ValueError
- exceptions are caught, so that the connection is not closed, only
- logging warnings.
- """
+ Same as Twisted's private _sslverify.ClientTLSOptions,
+ except that VerificationError, CertificateError and ValueError
+ exceptions are caught, so that the connection is not closed, only
+ logging warnings. Also, HTTPS connection parameters logging is added.
+ """
- def _identityVerifyingInfoCallback(self, connection, where, ret):
- if where & SSL_CB_HANDSHAKE_START:
- set_tlsext_host_name(connection, self._hostnameBytes)
- elif where & SSL_CB_HANDSHAKE_DONE:
- try:
- verifyHostname(connection, self._hostnameASCII)
- except verification_errors as e:
- logger.warning(
- 'Remote certificate is not valid for hostname "{}"; {}'.format(
- self._hostnameASCII, e))
+ def __init__(self, hostname, ctx, verbose_logging=False):
+ super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+ self.verbose_logging = verbose_logging
- except ValueError as e:
- logger.warning(
- 'Ignoring error while verifying certificate '
- 'from host "{}" (exception: {})'.format(
- self._hostnameASCII, repr(e)))
+ def _identityVerifyingInfoCallback(self, connection, where, ret):
+ if where & SSL.SSL_CB_HANDSHAKE_START:
+ set_tlsext_host_name(connection, self._hostnameBytes)
+ elif where & SSL.SSL_CB_HANDSHAKE_DONE:
+ if self.verbose_logging:
+ if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
+ if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
+ logger.debug('SSL connection to %s using protocol %s, cipher %s',
+ self._hostnameASCII,
+ connection.get_protocol_version_name(),
+ connection.get_cipher_name(),
+ )
+ else:
+ logger.debug('SSL connection to %s using cipher %s',
+ self._hostnameASCII,
+ connection.get_cipher_name(),
+ )
+ server_cert = connection.get_peer_certificate()
+ logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
+ x509name_to_string(server_cert.get_issuer()),
+ x509name_to_string(server_cert.get_subject()),
+ )
+ key_info = get_temp_key_info(connection._ssl)
+ if key_info:
+ logger.debug('SSL temp key: %s', key_info)
- DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
+ try:
+ verifyHostname(connection, self._hostnameASCII)
+ except (CertificateError, VerificationError) as e:
+ logger.warning(
+ 'Remote certificate is not valid for hostname "{}"; {}'.format(
+ self._hostnameASCII, e))
+
+ except ValueError as e:
+ logger.warning(
+ 'Ignoring error while verifying certificate '
+ 'from host "{}" (exception: {})'.format(
+ self._hostnameASCII, repr(e)))
+
+DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 1c89a0f9e..fc796e8bb 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,5 @@
from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
@@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b'\n'
def connectionMade(self):
- self.headers = Headers() # bucket for response headers
+ self.headers = Headers() # bucket for response headers
# Method command
self.sendCommand(self.factory.method, self.factory.path)
@@ -88,14 +88,14 @@ class ScrapyHTTPPageGetter(HTTPClient):
if self.factory.url.startswith(b'https'):
self.transport.stopProducing()
- self.factory.noPage(\
- defer.TimeoutError("Getting %s took longer than %s seconds." % \
- (self.factory.url, self.factory.timeout)))
+ self.factory.noPage(
+ defer.TimeoutError("Getting %s took longer than %s seconds." %
+ (self.factory.url, self.factory.timeout)))
class ScrapyHTTPClientFactory(HTTPClientFactory):
"""Scrapy implementation of the HTTPClientFactory overwriting the
- serUrl method to make use of our Url object that cache the parse
+ setUrl method to make use of our Url object that cache the parse
result.
"""
@@ -157,4 +157,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers
-
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 9d7552774..268ac75a5 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -25,7 +25,7 @@ class Slot(object):
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
- self.inprogress = set() # requests in progress
+ self.inprogress = set() # requests in progress
self.start_requests = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall
@@ -236,10 +236,11 @@ class ExecutionEngine(object):
def _on_success(response):
assert isinstance(response, (Response, Request))
if isinstance(response, Response):
- response.request = request # tie request to response received
+ response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
- self.signals.send_catch_log(signal=signals.response_received, \
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ self.signals.send_catch_log(signal=signals.response_received,
response=response, request=request, spider=spider)
return response
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a54b4daf0..975aede0c 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -1,19 +1,46 @@
import os
import json
import logging
+import warnings
from os.path import join, exists
-from scrapy.utils.reqser import request_to_dict, request_from_dict
-from scrapy.utils.misc import load_object
+from queuelib import PriorityQueue
+
+from scrapy.utils.misc import load_object, create_instance
from scrapy.utils.job import job_dir
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+
logger = logging.getLogger(__name__)
class Scheduler(object):
+ """
+ Scrapy Scheduler. It allows to enqueue requests and then get
+ a next request to download. Scheduler is also handling duplication
+ filtering, via dupefilter.
+ Prioritization and queueing is not performed by the Scheduler.
+ User sets ``priority`` field for each Request, and a PriorityQueue
+ (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
+ to dequeue requests in a desired order.
+
+ Scheduler uses two PriorityQueue instances, configured to work in-memory
+ and on-disk (optional). When on-disk queue is present, it is used by
+ default, and an in-memory queue is used as a fallback for cases where
+ a disk queue can't handle a request (can't serialize it).
+
+ :setting:`SCHEDULER_MEMORY_QUEUE` and
+ :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
+ which PriorityQueue instances would be instantiated with, to keep requests
+ on disk and in memory respectively.
+
+ Overall, Scheduler is an object which holds several PriorityQueue instances
+ (in-memory and on-disk) and implements fallback logic for them.
+ Also, it handles dupefilters.
+ """
def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
- logunser=False, stats=None, pqclass=None):
+ logunser=False, stats=None, pqclass=None, crawler=None):
self.df = dupefilter
self.dqdir = self._dqdir(jobdir)
self.pqclass = pqclass
@@ -21,33 +48,43 @@ class Scheduler(object):
self.mqclass = mqclass
self.logunser = logunser
self.stats = stats
+ self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
- dupefilter = dupefilter_cls.from_settings(settings)
+ dupefilter = create_instance(dupefilter_cls, settings, crawler)
pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
+ if pqclass is PriorityQueue:
+ warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
+ " is no longer supported because of API changes; "
+ "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
+ ScrapyDeprecationWarning)
+ from scrapy.pqueues import ScrapyPriorityQueue
+ pqclass = ScrapyPriorityQueue
+
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
- logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG'))
+ logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
+ settings.getbool('SCHEDULER_DEBUG'))
return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
- stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
+ stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
+ mqclass=mqclass, crawler=crawler)
def has_pending_requests(self):
return len(self) > 0
def open(self, spider):
self.spider = spider
- self.mqs = self.pqclass(self._newmq)
+ self.mqs = self._mq()
self.dqs = self._dq() if self.dqdir else None
return self.df.open()
def close(self, reason):
if self.dqs:
- prios = self.dqs.close()
- with open(join(self.dqdir, 'active.json'), 'w') as f:
- json.dump(prios, f)
+ state = self.dqs.close()
+ self._write_dqs_state(self.dqdir, state)
return self.df.close(reason)
def enqueue_request(self, request):
@@ -82,8 +119,7 @@ class Scheduler(object):
if self.dqs is None:
return
try:
- reqd = request_to_dict(request, self.spider)
- self.dqs.push(reqd, -request.priority)
+ self.dqs.push(request, -request.priority)
except ValueError as e: # non serializable request
if self.logunser:
msg = ("Unable to serialize request: %(request)s - reason:"
@@ -103,32 +139,51 @@ class Scheduler(object):
def _dqpop(self):
if self.dqs:
- d = self.dqs.pop()
- if d:
- return request_from_dict(d, self.spider)
+ return self.dqs.pop()
def _newmq(self, priority):
+ """ Factory for creating memory queues. """
return self.mqclass()
def _newdq(self, priority):
- return self.dqclass(join(self.dqdir, 'p%s' % priority))
+ """ Factory for creating disk queues. """
+ path = join(self.dqdir, 'p%s' % (priority, ))
+ return self.dqclass(path)
+
+ def _mq(self):
+ """ Create a new priority queue instance, with in-memory storage """
+ return create_instance(self.pqclass, None, self.crawler, self._newmq,
+ serialize=False)
def _dq(self):
- activef = join(self.dqdir, 'active.json')
- if exists(activef):
- with open(activef) as f:
- prios = json.load(f)
- else:
- prios = ()
- q = self.pqclass(self._newdq, startprios=prios)
+ """ Create a new priority queue instance, with disk storage """
+ state = self._read_dqs_state(self.dqdir)
+ q = create_instance(self.pqclass,
+ None,
+ self.crawler,
+ self._newdq,
+ state,
+ serialize=True)
if q:
logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
{'queuesize': len(q)}, extra={'spider': self.spider})
return q
def _dqdir(self, jobdir):
+ """ Return a folder name to keep disk queue state at """
if jobdir:
dqdir = join(jobdir, 'requests.queue')
if not exists(dqdir):
os.makedirs(dqdir)
return dqdir
+
+ def _read_dqs_state(self, dqdir):
+ path = join(dqdir, 'active.json')
+ if not exists(path):
+ return ()
+ with open(path) as f:
+ return json.load(f)
+
+ def _write_dqs_state(self, dqdir, state):
+ with open(join(dqdir, 'active.json'), 'w') as f:
+ json.dump(state, f)
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index c08e37367..facbd8b73 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -9,7 +9,7 @@ from twisted.internet import defer
from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
from scrapy.utils.spider import iterate_spider_output
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.log import logformatter_adapter, failure_to_exc_info
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
from scrapy import signals
@@ -18,6 +18,7 @@ from scrapy.item import BaseItem
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.utils.request import referer_str
+
logger = logging.getLogger(__name__)
@@ -77,7 +78,7 @@ class Scraper(object):
@defer.inlineCallbacks
def open_spider(self, spider):
"""Open the given spider for scraping and allocate resources for it"""
- self.slot = Slot()
+ self.slot = Slot(self.crawler.settings.getint('SCRAPER_SLOT_MAX_ACTIVE_SIZE'))
yield self.itemproc.open_spider(spider)
def close_spider(self, spider):
@@ -99,11 +100,13 @@ class Scraper(object):
def enqueue_scrape(self, response, request, spider):
slot = self.slot
dfd = slot.add_response_request(response, request)
+
def finish_scraping(_):
slot.finish_response(response, request)
self._check_if_closing(spider, slot)
self._scrape_next(spider, slot)
return _
+
dfd.addBoth(finish_scraping)
dfd.addErrback(
lambda f: logger.error('Scraper bug processing %(request)s',
@@ -123,7 +126,7 @@ class Scraper(object):
callback/errback"""
assert isinstance(response, (Response, Failure))
- dfd = self._scrape2(response, request, spider) # returns spiders processed output
+ dfd = self._scrape2(response, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
return dfd
@@ -135,7 +138,6 @@ class Scraper(object):
return self.spidermw.scrape_response(
self.call_spider, request_result, request, spider)
else:
- # FIXME: don't ignore errors in spider middleware
dfd = self.call_spider(request_result, request, spider)
return dfd.addErrback(
self._log_download_errors, request_result, request, spider)
@@ -143,7 +145,12 @@ class Scraper(object):
def call_spider(self, result, request, spider):
result.request = request
dfd = defer_result(result)
- dfd.addCallbacks(request.callback or spider.parse, request.errback)
+ callback = request.callback or spider.parse
+ warn_on_generator_with_return_value(spider, callback)
+ warn_on_generator_with_return_value(spider, request.errback)
+ dfd.addCallbacks(callback=callback,
+ errback=request.errback,
+ callbackKeywords=request.cb_kwargs)
return dfd.addCallback(iterate_spider_output)
def handle_spider_error(self, _failure, request, response, spider):
@@ -171,8 +178,8 @@ class Scraper(object):
if not result:
return defer_succeed(None)
it = iter_errback(result, self.handle_spider_error, request, response, spider)
- dfd = parallel(it, self.concurrent_items,
- self._process_spidermw_output, request, response, spider)
+ dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
+ request, response, spider)
return dfd
def _process_spidermw_output(self, output, request, response, spider):
@@ -199,8 +206,7 @@ class Scraper(object):
"""Log and silence errors that come from the engine (typically download
errors that got propagated thru here)
"""
- if (isinstance(download_failure, Failure) and
- not download_failure.check(IgnoreRequest)):
+ if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
if download_failure.frames:
logger.error('Error downloading %(request)s',
{'request': request},
@@ -224,18 +230,22 @@ class Scraper(object):
ex = output.value
if isinstance(ex, DropItem):
logkws = self.logformatter.dropped(item, ex, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
- logger.error('Error processing %(item)s', {'item': item},
- exc_info=failure_to_exc_info(output),
- extra={'spider': spider})
+ logkws = self.logformatter.error(item, ex, response, spider)
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
+ exc_info=failure_to_exc_info(output))
+ return self.signals.send_catch_log_deferred(
+ signal=signals.item_error, item=item, response=response,
+ spider=spider, failure=output)
else:
logkws = self.logformatter.scraped(output, response, spider)
- logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
+ if logkws is not None:
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
return self.signals.send_catch_log_deferred(
signal=signals.item_scraped, item=output, response=response,
spider=spider)
-
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index a206e4b0c..097a374bf 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,15 +3,20 @@ Spider Middleware manager
See documentation in docs/topics/spider-middleware.rst
"""
-import six
+from itertools import chain, islice
+
from twisted.python.failure import Failure
+from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.conf import build_component_list
+from scrapy.utils.python import MutableChain
+
def _isiterable(possible_iterator):
return hasattr(possible_iterator, '__iter__')
+
class SpiderMiddlewareManager(MiddlewareManager):
component_name = 'spider middleware'
@@ -24,52 +29,87 @@ class SpiderMiddlewareManager(MiddlewareManager):
super(SpiderMiddlewareManager, self)._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
- if hasattr(mw, 'process_spider_output'):
- self.methods['process_spider_output'].insert(0, mw.process_spider_output)
- if hasattr(mw, 'process_spider_exception'):
- self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)
if hasattr(mw, 'process_start_requests'):
- self.methods['process_start_requests'].insert(0, mw.process_start_requests)
+ self.methods['process_start_requests'].appendleft(mw.process_start_requests)
+ self.methods['process_spider_output'].appendleft(getattr(mw, 'process_spider_output', None))
+ self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
def scrape_response(self, scrape_func, response, request, spider):
- fname = lambda f:'%s.%s' % (
- six.get_method_self(f).__class__.__name__,
- six.get_method_function(f).__name__)
+ fname = lambda f: '%s.%s' % (
+ f.__self__.__class__.__name__,
+ f.__func__.__name__)
def process_spider_input(response):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
- assert result is None, \
- 'Middleware %s must returns None or ' \
- 'raise an exception, got %s ' \
- % (fname(method), type(result))
- except:
+ if result is not None:
+ raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
+ .format(fname(method), type(result)))
+ except _InvalidOutput:
+ raise
+ except Exception:
return scrape_func(Failure(), request, spider)
return scrape_func(response, request, spider)
- def process_spider_exception(_failure):
+ def process_spider_exception(_failure, start_index=0):
exception = _failure.value
- for method in self.methods['process_spider_exception']:
+ # don't handle _InvalidOutput exception
+ if isinstance(exception, _InvalidOutput):
+ return _failure
+ method_list = islice(self.methods['process_spider_exception'], start_index, None)
+ for method_index, method in enumerate(method_list, start=start_index):
+ if method is None:
+ continue
result = method(response=response, exception=exception, spider=spider)
- assert result is None or _isiterable(result), \
- 'Middleware %s must returns None, or an iterable object, got %s ' % \
- (fname(method), type(result))
- if result is not None:
- return result
+ if _isiterable(result):
+ # stop exception handling by handing control over to the
+ # process_spider_output chain if an iterable has been returned
+ return process_spider_output(result, method_index+1)
+ elif result is None:
+ continue
+ else:
+ raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
+ .format(fname(method), type(result)))
return _failure
- def process_spider_output(result):
- for method in self.methods['process_spider_output']:
- result = method(response=response, result=result, spider=spider)
- assert _isiterable(result), \
- 'Middleware %s must returns an iterable object, got %s ' % \
- (fname(method), type(result))
- return result
+ def process_spider_output(result, start_index=0):
+ # items in this iterable do not need to go through the process_spider_output
+ # chain, they went through it already from the process_spider_exception method
+ recovered = MutableChain()
+
+ def evaluate_iterable(iterable, index):
+ try:
+ for r in iterable:
+ yield r
+ except Exception as ex:
+ exception_result = process_spider_exception(Failure(ex), index+1)
+ if isinstance(exception_result, Failure):
+ raise
+ recovered.extend(exception_result)
+
+ method_list = islice(self.methods['process_spider_output'], start_index, None)
+ for method_index, method in enumerate(method_list, start=start_index):
+ if method is None:
+ continue
+ # the following might fail directly if the output value is not a generator
+ try:
+ result = method(response=response, result=result, spider=spider)
+ except Exception as ex:
+ exception_result = process_spider_exception(Failure(ex), method_index+1)
+ if isinstance(exception_result, Failure):
+ raise
+ return exception_result
+ if _isiterable(result):
+ result = evaluate_iterable(result, method_index)
+ else:
+ raise _InvalidOutput('Middleware {} must return an iterable, got {}'
+ .format(fname(method), type(result)))
+
+ return chain(result, recovered)
dfd = mustbe_deferred(process_spider_input, response)
- dfd.addErrback(process_spider_exception)
- dfd.addCallback(process_spider_output)
+ dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception)
return dfd
def process_start_requests(self, start_requests, spider):
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index d05c05d28..54fba4ac1 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,32 +1,39 @@
-import six
-import signal
import logging
+import pprint
+import signal
import warnings
-import sys
-from twisted.internet import reactor, defer
-from zope.interface.verify import verifyClass, DoesNotImplement
+from twisted.internet import defer
+from zope.interface.verify import DoesNotImplement, verifyClass
+from scrapy import signals, Spider
from scrapy.core.engine import ExecutionEngine
-from scrapy.resolver import CachingThreadedResolver
-from scrapy.interfaces import ISpiderLoader
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extension import ExtensionManager
+from scrapy.interfaces import ISpiderLoader
from scrapy.settings import overridden_settings, Settings
from scrapy.signalmanager import SignalManager
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.misc import load_object
+from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
from scrapy.utils.log import (
- LogCounterHandler, configure_logging, log_scrapy_info,
- get_scrapy_root_handler, install_scrapy_root_handler)
-from scrapy import signals
+ configure_logging,
+ get_scrapy_root_handler,
+ install_scrapy_root_handler,
+ log_scrapy_info,
+ LogCounterHandler,
+)
+from scrapy.utils.misc import create_instance, load_object
+from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
+
logger = logging.getLogger(__name__)
-class Crawler(object):
+class Crawler:
def __init__(self, spidercls, settings=None):
+ if isinstance(spidercls, Spider):
+ raise ValueError('The spidercls argument must be a class, not an object')
+
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)
@@ -34,14 +41,16 @@ class Crawler(object):
self.settings = settings.copy()
self.spidercls.update_settings(self.settings)
- d = dict(overridden_settings(self.settings))
- logger.info("Overridden settings: %(settings)r", {'settings': d})
-
self.signals = SignalManager(self)
self.stats = load_object(self.settings['STATS_CLASS'])(self)
handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
logging.root.addHandler(handler)
+
+ d = dict(overridden_settings(self.settings))
+ logger.info("Overridden settings:\n%(settings)s",
+ {'settings': pprint.pformat(d)})
+
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
install_scrapy_root_handler(self.settings)
@@ -82,20 +91,9 @@ class Crawler(object):
yield self.engine.open_spider(self.spider, start_requests)
yield defer.maybeDeferred(self.engine.start)
except Exception:
- # In Python 2 reraising an exception after yield discards
- # the original traceback (see https://bugs.python.org/issue7563),
- # so sys.exc_info() workaround is used.
- # This workaround also works in Python 3, but it is not needed,
- # and it is slower, so in Python 3 we use native `raise`.
- if six.PY2:
- exc_info = sys.exc_info()
-
self.crawling = False
if self.engine is not None:
yield self.engine.close()
-
- if six.PY2:
- six.reraise(*exc_info)
raise
def _create_spider(self, *args, **kwargs):
@@ -106,15 +104,17 @@ class Crawler(object):
@defer.inlineCallbacks
def stop(self):
+ """Starts a graceful stop of the crawler and returns a deferred that is
+ fired when the crawler is stopped."""
if self.crawling:
self.crawling = False
yield defer.maybeDeferred(self.engine.stop)
-class CrawlerRunner(object):
+class CrawlerRunner:
"""
This is a convenient helper class that keeps track of, manages and runs
- crawlers inside an already setup Twisted `reactor`_.
+ crawlers inside an already setup :mod:`~twisted.internet.reactor`.
The CrawlerRunner object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
@@ -137,6 +137,8 @@ class CrawlerRunner(object):
self.spider_loader = _get_spider_loader(settings)
self._crawlers = set()
self._active = set()
+ self.bootstrap_failed = False
+ self._handle_asyncio_reactor()
@property
def spiders(self):
@@ -152,7 +154,7 @@ class CrawlerRunner(object):
It will call the given Crawler's :meth:`~Crawler.crawl` method, while
keeping track of it so it can be stopped later.
- If `crawler_or_spidercls` isn't a :class:`~scrapy.crawler.Crawler`
+ If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
instance, this method will try to create one using this parameter as
the spider class given to it.
@@ -167,6 +169,10 @@ class CrawlerRunner(object):
:param dict kwargs: keyword arguments to initialize the spider
"""
+ if isinstance(crawler_or_spidercls, Spider):
+ raise ValueError(
+ 'The crawler_or_spidercls argument cannot be a spider object, '
+ 'it must be a spider class (or a Crawler object)')
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
@@ -178,6 +184,7 @@ class CrawlerRunner(object):
def _done(result):
self.crawlers.discard(crawler)
self._active.discard(d)
+ self.bootstrap_failed |= not getattr(crawler, 'spider', None)
return result
return d.addBoth(_done)
@@ -186,19 +193,23 @@ class CrawlerRunner(object):
"""
Return a :class:`~scrapy.crawler.Crawler` object.
- * If `crawler_or_spidercls` is a Crawler, it is returned as-is.
- * If `crawler_or_spidercls` is a Spider subclass, a new Crawler
+ * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is.
+ * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler
is constructed for it.
- * If `crawler_or_spidercls` is a string, this function finds
+ * If ``crawler_or_spidercls`` is a string, this function finds
a spider with this name in a Scrapy project (using spider loader),
then creates a Crawler instance for it.
"""
+ if isinstance(crawler_or_spidercls, Spider):
+ raise ValueError(
+ 'The crawler_or_spidercls argument cannot be a spider object, '
+ 'it must be a spider class (or a Crawler object)')
if isinstance(crawler_or_spidercls, Crawler):
return crawler_or_spidercls
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
- if isinstance(spidercls, six.string_types):
+ if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)
@@ -221,18 +232,24 @@ class CrawlerRunner(object):
while self._active:
yield defer.DeferredList(self._active)
+ def _handle_asyncio_reactor(self):
+ if self.settings.getbool('ASYNCIO_REACTOR') and not is_asyncio_reactor_installed():
+ raise Exception("ASYNCIO_REACTOR is on but the Twisted asyncio "
+ "reactor is not installed.")
+
class CrawlerProcess(CrawlerRunner):
"""
A class to run multiple scrapy crawlers in a process simultaneously.
This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
- for starting a Twisted `reactor`_ and handling shutdown signals, like the
- keyboard interrupt command Ctrl-C. It also configures top-level logging.
+ for starting a :mod:`~twisted.internet.reactor` and handling shutdown
+ signals, like the keyboard interrupt command Ctrl-C. It also configures
+ top-level logging.
This utility should be a better fit than
:class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
- Twisted `reactor`_ within your application.
+ :mod:`~twisted.internet.reactor` within your application.
The CrawlerProcess object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
@@ -252,6 +269,7 @@ class CrawlerProcess(CrawlerRunner):
log_scrapy_info(self.settings)
def _signal_shutdown(self, signum, _):
+ from twisted.internet import reactor
install_shutdown_handlers(self._signal_kill)
signame = signal_names[signum]
logger.info("Received %(signame)s, shutting down gracefully. Send again to force ",
@@ -259,6 +277,7 @@ class CrawlerProcess(CrawlerRunner):
reactor.callFromThread(self._graceful_stop_reactor)
def _signal_kill(self, signum, _):
+ from twisted.internet import reactor
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
logger.info('Received %(signame)s twice, forcing unclean shutdown',
@@ -267,16 +286,17 @@ class CrawlerProcess(CrawlerRunner):
def start(self, stop_after_crawl=True):
"""
- This method starts a Twisted `reactor`_, adjusts its pool size to
- :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
- on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
+ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
+ size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
+ based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
- If `stop_after_crawl` is True, the reactor will be stopped after all
+ If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
:param boolean stop_after_crawl: stop or not the reactor when all
crawlers have finished
"""
+ from twisted.internet import reactor
if stop_after_crawl:
d = self.join()
# Don't start the reactor if the deferreds are already fired
@@ -284,45 +304,35 @@ class CrawlerProcess(CrawlerRunner):
return
d.addBoth(self._stop_reactor)
- reactor.installResolver(self._get_dns_resolver())
+ resolver_class = load_object(self.settings["DNS_RESOLVER"])
+ resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
+ resolver.install_on_reactor()
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run(installSignalHandlers=False) # blocking call
- def _get_dns_resolver(self):
- if self.settings.getbool('DNSCACHE_ENABLED'):
- cache_size = self.settings.getint('DNSCACHE_SIZE')
- else:
- cache_size = 0
- return CachingThreadedResolver(
- reactor=reactor,
- cache_size=cache_size,
- timeout=self.settings.getfloat('DNS_TIMEOUT')
- )
-
def _graceful_stop_reactor(self):
d = self.stop()
d.addBoth(self._stop_reactor)
return d
def _stop_reactor(self, _=None):
+ from twisted.internet import reactor
try:
reactor.stop()
except RuntimeError: # raised if already stopped or in shutdown stage
pass
+ def _handle_asyncio_reactor(self):
+ if self.settings.getbool('ASYNCIO_REACTOR'):
+ install_asyncio_reactor()
+ super()._handle_asyncio_reactor()
+
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
- if settings.get('SPIDER_MANAGER_CLASS'):
- warnings.warn(
- 'SPIDER_MANAGER_CLASS option is deprecated. '
- 'Please use SPIDER_LOADER_CLASS.',
- category=ScrapyDeprecationWarning, stacklevel=2
- )
- cls_path = settings.get('SPIDER_MANAGER_CLASS',
- settings.get('SPIDER_LOADER_CLASS'))
+ cls_path = settings.get('SPIDER_LOADER_CLASS')
loader_cls = load_object(cls_path)
try:
verifyClass(ISpiderLoader, loader_cls)
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 72715dba7..7a140fcad 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
import re
import logging
-import six
from w3lib import html
from scrapy.exceptions import NotConfigured
@@ -67,7 +65,9 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
+_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
+
+
def _has_ajaxcrawlable_meta(text):
"""
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 321c0171b..d8dabdf13 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,12 +1,11 @@
-import os
-import six
import logging
from collections import defaultdict
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
+
logger = logging.getLogger(__name__)
@@ -53,7 +52,7 @@ class CookiesMiddleware(object):
def _debug_cookie(self, request, spider):
if self.debug:
- cl = [to_native_str(c, errors='replace')
+ cl = [to_unicode(c, errors='replace')
for c in request.headers.getlist('Cookie')]
if cl:
cookies = "\n".join("Cookie: {}\n".format(c) for c in cl)
@@ -62,7 +61,7 @@ class CookiesMiddleware(object):
def _debug_set_cookie(self, response, spider):
if self.debug:
- cl = [to_native_str(c, errors='replace')
+ cl = [to_unicode(c, errors='replace')
for c in response.headers.getlist('Set-Cookie')]
if cl:
cookies = "\n".join("Set-Cookie: {}\n".format(c) for c in cl)
@@ -82,8 +81,10 @@ class CookiesMiddleware(object):
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
- cookie_list = [{'name': k, 'value': v} for k, v in \
- six.iteritems(request.cookies)]
+ cookie_list = [
+ {'name': k, 'value': v}
+ for k, v in request.cookies.items()
+ ]
else:
cookie_list = request.cookies
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 49313cc04..fcea38ef5 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,20 +4,15 @@ and extract the potentially compressed responses that may arrive.
import bz2
import gzip
-import zipfile
-import tarfile
import logging
+import tarfile
+import zipfile
+from io import BytesIO
from tempfile import mktemp
-import six
-
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
-
from scrapy.responsetypes import responsetypes
+
logger = logging.getLogger(__name__)
@@ -79,7 +74,7 @@ class DecompressionMiddleware(object):
if not response.body:
return response
- for fmt, func in six.iteritems(self._formats):
+ for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug('Decompressed response with format: %(responsefmt)s',
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 495b103d1..4e06f8236 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,11 +1,19 @@
from email.utils import formatdate
+
from twisted.internet import defer
-from twisted.internet.error import TimeoutError, DNSLookupError, \
- ConnectionRefusedError, ConnectionDone, ConnectError, \
- ConnectionLost, TCPTimedOutError
+from twisted.internet.error import (
+ ConnectError,
+ ConnectionDone,
+ ConnectionLost,
+ ConnectionRefusedError,
+ DNSLookupError,
+ TCPTimedOutError,
+ TimeoutError,
+)
from twisted.web.client import ResponseFailed
+
from scrapy import signals
-from scrapy.exceptions import NotConfigured, IgnoreRequest
+from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.utils.misc import load_object
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 203dee42d..0010b2a8f 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -26,7 +26,7 @@ class HttpCompressionMiddleware(object):
def process_request(self, request, spider):
request.headers.setdefault('Accept-Encoding',
- b",".join(ACCEPTED_ENCODINGS))
+ b", ".join(ACCEPTED_ENCODINGS))
def process_response(self, request, response, spider):
@@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object):
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
- respcls = responsetypes.from_args(headers=response.headers, \
- url=response.url, body=decoded_body)
+ respcls = responsetypes.from_args(
+ headers=response.headers, url=response.url, body=decoded_body
+ )
kwargs = dict(cls=respcls, body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 0d5320bf8..814ce78fe 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,14 +1,9 @@
import base64
-from six.moves.urllib.request import getproxies, proxy_bypass
-from six.moves.urllib.parse import unquote
-try:
- from urllib2 import _parse_proxy
-except ImportError:
- from urllib.request import _parse_proxy
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import unquote, urlunparse
+from urllib.request import getproxies, proxy_bypass, _parse_proxy
-from scrapy.utils.httpobj import urlparse_cached
from scrapy.exceptions import NotConfigured
+from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes
@@ -17,8 +12,8 @@ class HttpProxyMiddleware(object):
def __init__(self, auth_encoding='latin-1'):
self.auth_encoding = auth_encoding
self.proxies = {}
- for type, url in getproxies().items():
- self.proxies[type] = self._get_proxy(url, type)
+ for type_, url in getproxies().items():
+ self.proxies[type_] = self._get_proxy(url, type_)
@classmethod
def from_crawler(cls, crawler):
@@ -31,7 +26,7 @@ class HttpProxyMiddleware(object):
user_pass = to_bytes(
'%s:%s' % (unquote(username), unquote(password)),
encoding=self.auth_encoding)
- return base64.b64encode(user_pass).strip()
+ return base64.b64encode(user_pass)
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 30cae3fee..77cb5aa94 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
import logging
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
@@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
+
logger = logging.getLogger(__name__)
@@ -34,6 +35,8 @@ class BaseRedirectMiddleware(object):
redirected.meta['redirect_ttl'] = ttl - 1
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
[request.url]
+ redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
+ [reason]
redirected.dont_filter = request.dont_filter
redirected.priority = request.priority + self.priority_adjust
logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
@@ -68,7 +71,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if 'Location' not in response.headers or response.status not in allowed_status:
return response
- location = safe_url_string(response.headers['location'])
+ location = safe_url_string(response.headers['Location'])
+ if response.headers['Location'].startswith(b'//'):
+ request_scheme = urlparse(request.url).scheme
+ location = request_scheme + '://' + location.lstrip('/')
redirected_url = urljoin(request.url, location)
@@ -86,6 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
def __init__(self, settings):
super(MetaRefreshMiddleware, self).__init__(settings)
+ self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',
settings.getint('METAREFRESH_MAXDELAY'))
@@ -94,7 +101,8 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
not isinstance(response, HtmlResponse):
return response
- interval, url = get_meta_refresh(response)
+ interval, url = get_meta_refresh(response,
+ ignore_tags=self._ignore_tags)
if url and interval < self._maxdelay:
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, 'meta refresh')
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 07e979628..7ab5b6e62 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -7,9 +7,7 @@ RETRY_TIMES - how many times to retry a failed page
RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
-once the spider has finished crawling all regular (non failed) pages. Once
-there is no more failed pages to retry this middleware sends a signal
-(retry_complete), so other extensions could connect to that signal.
+once the spider has finished crawling all regular (non failed) pages.
"""
import logging
@@ -86,6 +84,6 @@ class RetryMiddleware(object):
return retryreq
else:
stats.inc_value('retry/max_reached')
- logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
+ logger.error("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",
{'request': request, 'retries': retries, 'reason': reason},
extra={'spider': spider})
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 200245210..251706c50 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -6,14 +6,12 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
import logging
-from six.moves.urllib import robotparser
-
from twisted.internet.defer import Deferred, maybeDeferred
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_native_str
+from scrapy.utils.misc import load_object
logger = logging.getLogger(__name__)
@@ -24,10 +22,14 @@ class RobotsTxtMiddleware(object):
def __init__(self, crawler):
if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
raise NotConfigured
-
+ self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
+ self._robotstxt_useragent = crawler.settings.get('ROBOTSTXT_USER_AGENT', None)
self.crawler = crawler
- self._useragent = crawler.settings.get('USER_AGENT')
self._parsers = {}
+ self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
+
+ # check if parser dependencies are met, this should throw an error otherwise.
+ self._parserimpl.from_crawler(self.crawler, b'')
@classmethod
def from_crawler(cls, crawler):
@@ -43,7 +45,11 @@ class RobotsTxtMiddleware(object):
def process_request_2(self, rp, request, spider):
if rp is None:
return
- if not rp.can_fetch(to_native_str(self._useragent), request.url):
+
+ useragent = self._robotstxt_useragent
+ if not useragent:
+ useragent = request.headers.get(b'User-Agent', self._default_useragent)
+ if not rp.allowed(request.url, useragent):
logger.debug("Forbidden by robots.txt: %(request)s",
{'request': request}, extra={'spider': spider})
self.crawler.stats.inc_value('robotstxt/forbidden')
@@ -62,13 +68,14 @@ class RobotsTxtMiddleware(object):
meta={'dont_obey_robotstxt': True}
)
dfd = self.crawler.engine.download(robotsreq, spider)
- dfd.addCallback(self._parse_robots, netloc)
+ dfd.addCallback(self._parse_robots, netloc, spider)
dfd.addErrback(self._logerror, robotsreq, spider)
dfd.addErrback(self._robots_error, netloc)
self.crawler.stats.inc_value('robotstxt/request_count')
if isinstance(self._parsers[netloc], Deferred):
d = Deferred()
+
def cb(result):
d.callback(result)
return result
@@ -85,27 +92,10 @@ class RobotsTxtMiddleware(object):
extra={'spider': spider})
return failure
- def _parse_robots(self, response, netloc):
+ def _parse_robots(self, response, netloc, spider):
self.crawler.stats.inc_value('robotstxt/response_count')
- self.crawler.stats.inc_value(
- 'robotstxt/response_status_count/{}'.format(response.status))
- rp = robotparser.RobotFileParser(response.url)
- body = ''
- if hasattr(response, 'text'):
- body = response.text
- else: # last effort try
- try:
- body = response.body.decode('utf-8')
- except UnicodeDecodeError:
- # If we found garbage, disregard it:,
- # but keep the lookup cached (in self._parsers)
- # Running rp.parse() will set rp state from
- # 'disallow all' to 'allow any'.
- self.crawler.stats.inc_value('robotstxt/unicode_error_count')
- # stdlib's robotparser expects native 'str' ;
- # with unicode input, non-ASCII encoded bytes decoding fails in Python2
- rp.parse(to_native_str(body).splitlines())
-
+ self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
+ rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp_dfd = self._parsers[netloc]
self._parsers[netloc] = rp
rp_dfd.callback(rp)
diff --git a/scrapy/dupefilter.py b/scrapy/dupefilter.py
deleted file mode 100644
index 232d96288..000000000
--- a/scrapy/dupefilter.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.dupefilter` is deprecated, "
- "use `scrapy.dupefilters` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.dupefilters import *
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 9d8966b9c..ea6a4cfc3 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,9 +1,8 @@
-from __future__ import print_function
import os
import logging
from scrapy.utils.job import job_dir
-from scrapy.utils.request import request_fingerprint
+from scrapy.utils.request import referer_str, request_fingerprint
class BaseDupeFilter(object):
@@ -61,8 +60,9 @@ class RFPDupeFilter(BaseDupeFilter):
def log(self, request, spider):
if self.debug:
- msg = "Filtered duplicate request: %(request)s"
- self.logger.debug(msg, {'request': request}, extra={'spider': spider})
+ msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
+ args = {'request': request, 'referer': referer_str(request) }
+ self.logger.debug(msg, args, extra={'spider': spider})
elif self.logdupes:
msg = ("Filtered duplicate request: %(request)s"
" - no more duplicates will be shown"
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 4bcecd994..7c4bb3d00 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -7,19 +7,32 @@ new exceptions here without documenting them there.
# Internal
+
class NotConfigured(Exception):
"""Indicates a missing configuration situation"""
pass
+
+class _InvalidOutput(TypeError):
+ """
+ Indicates an invalid value has been returned by a middleware's processing method.
+ Internal and undocumented, it should not be raised or caught by user code.
+ """
+ pass
+
+
# HTTP and crawling
+
class IgnoreRequest(Exception):
"""Indicates a decision was made not to process a request"""
+
class DontCloseSpider(Exception):
"""Request the spider not to be closed yet"""
pass
+
class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
@@ -27,30 +40,37 @@ class CloseSpider(Exception):
super(CloseSpider, self).__init__()
self.reason = reason
+
# Items
+
class DropItem(Exception):
"""Drop item from the item pipeline"""
pass
+
class NotSupported(Exception):
"""Indicates a feature or method is not supported"""
pass
+
# Commands
+
class UsageError(Exception):
"""To indicate a command-line usage error"""
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)
+
class ScrapyDeprecationWarning(Warning):
"""Warning category for deprecated features, since the default
DeprecationWarning is silenced on Python 2.7+
"""
pass
+
class ContractFail(AssertionError):
"""Error raised in case of a failing contract"""
pass
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 07f43b494..1a3c9345f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -4,18 +4,16 @@ Item Exporters are used to export/serialize items into different formats.
import csv
import io
-import sys
import pprint
import marshal
-import six
-from six.moves import cPickle as pickle
+import warnings
+import pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
-from scrapy.utils.python import to_bytes, to_unicode, to_native_str, is_listlike
+from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import BaseItem
from scrapy.exceptions import ScrapyDeprecationWarning
-import warnings
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@@ -25,13 +23,14 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
class BaseItemExporter(object):
- def __init__(self, **kwargs):
- self._configure(kwargs)
+ def __init__(self, dont_fail=False, **kwargs):
+ self._kwargs = kwargs
+ self._configure(kwargs, dont_fail=dont_fail)
def _configure(self, options, dont_fail=False):
"""Configure the exporter by poping options from the ``options`` dict.
If dont_fail is set, it won't raise an exception on unexpected options
- (useful for using with keyword arguments in subclasses constructors)
+ (useful for using with keyword arguments in subclasses ``__init__`` methods)
"""
self.encoding = options.pop('encoding', None)
self.fields_to_export = options.pop('fields_to_export', None)
@@ -61,9 +60,9 @@ class BaseItemExporter(object):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
if include_empty and not isinstance(item, dict):
- field_iter = six.iterkeys(item.fields)
+ field_iter = item.fields.keys()
else:
- field_iter = six.iterkeys(item)
+ field_iter = item.keys()
else:
if include_empty:
field_iter = self.fields_to_export
@@ -83,10 +82,10 @@ class BaseItemExporter(object):
class JsonLinesItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
- self._configure(kwargs, dont_fail=True)
+ super().__init__(dont_fail=True, **kwargs)
self.file = file
- kwargs.setdefault('ensure_ascii', not self.encoding)
- self.encoder = ScrapyJSONEncoder(**kwargs)
+ self._kwargs.setdefault('ensure_ascii', not self.encoding)
+ self.encoder = ScrapyJSONEncoder(**self._kwargs)
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
@@ -97,15 +96,15 @@ class JsonLinesItemExporter(BaseItemExporter):
class JsonItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
- self._configure(kwargs, dont_fail=True)
+ super().__init__(dont_fail=True, **kwargs)
self.file = file
# there is a small difference between the behaviour or JsonItemExporter.indent
# and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
# the addition of newlines everywhere
json_indent = self.indent if self.indent is not None and self.indent > 0 else None
- kwargs.setdefault('indent', json_indent)
- kwargs.setdefault('ensure_ascii', not self.encoding)
- self.encoder = ScrapyJSONEncoder(**kwargs)
+ self._kwargs.setdefault('indent', json_indent)
+ self._kwargs.setdefault('ensure_ascii', not self.encoding)
+ self.encoder = ScrapyJSONEncoder(**self._kwargs)
self.first_item = True
def _beautify_newline(self):
@@ -136,18 +135,18 @@ class XmlItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
self.item_element = kwargs.pop('item_element', 'item')
self.root_element = kwargs.pop('root_element', 'items')
- self._configure(kwargs)
+ super().__init__(**kwargs)
if not self.encoding:
self.encoding = 'utf-8'
self.xg = XMLGenerator(file, encoding=self.encoding)
def _beautify_newline(self, new_item=False):
if self.indent is not None and (self.indent > 0 or new_item):
- self._xg_characters('\n')
+ self.xg.characters('\n')
def _beautify_indent(self, depth=1):
if self.indent:
- self._xg_characters(' ' * self.indent * depth)
+ self.xg.characters(' ' * self.indent * depth)
def start_exporting(self):
self.xg.startDocument()
@@ -181,32 +180,18 @@ class XmlItemExporter(BaseItemExporter):
for value in serialized_value:
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
- elif isinstance(serialized_value, six.text_type):
- self._xg_characters(serialized_value)
+ elif isinstance(serialized_value, str):
+ self.xg.characters(serialized_value)
else:
- self._xg_characters(str(serialized_value))
+ self.xg.characters(str(serialized_value))
self.xg.endElement(name)
self._beautify_newline()
- # Workaround for https://bugs.python.org/issue17606
- # Before Python 2.7.4 xml.sax.saxutils required bytes;
- # since 2.7.4 it requires unicode. The bug is likely to be
- # fixed in 2.7.6, but 2.7.6 will still support unicode,
- # and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
- if sys.version_info[:3] >= (2, 7, 4):
- def _xg_characters(self, serialized_value):
- if not isinstance(serialized_value, six.text_type):
- serialized_value = serialized_value.decode(self.encoding)
- return self.xg.characters(serialized_value)
- else: # pragma: no cover
- def _xg_characters(self, serialized_value):
- return self.xg.characters(serialized_value)
-
class CsvItemExporter(BaseItemExporter):
def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
- self._configure(kwargs, dont_fail=True)
+ super().__init__(dont_fail=True, **kwargs)
if not self.encoding:
self.encoding = 'utf-8'
self.include_headers_line = include_headers_line
@@ -214,9 +199,10 @@ class CsvItemExporter(BaseItemExporter):
file,
line_buffering=False,
write_through=True,
- encoding=self.encoding
- ) if six.PY3 else file
- self.csv_writer = csv.writer(self.stream, **kwargs)
+ encoding=self.encoding,
+ newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+ )
+ self.csv_writer = csv.writer(self.stream, **self._kwargs)
self._headers_not_written = True
self._join_multivalued = join_multivalued
@@ -245,7 +231,7 @@ class CsvItemExporter(BaseItemExporter):
def _build_row(self, values):
for s in values:
try:
- yield to_native_str(s, self.encoding)
+ yield to_unicode(s, self.encoding)
except TypeError:
yield s
@@ -265,7 +251,7 @@ class CsvItemExporter(BaseItemExporter):
class PickleItemExporter(BaseItemExporter):
def __init__(self, file, protocol=2, **kwargs):
- self._configure(kwargs)
+ super().__init__(**kwargs)
self.file = file
self.protocol = protocol
@@ -275,9 +261,16 @@ class PickleItemExporter(BaseItemExporter):
class MarshalItemExporter(BaseItemExporter):
+ """Exports items in a Python-specific binary format (see
+ :mod:`marshal`).
+
+ :param file: The file-like object to use for exporting the data. Its
+ ``write`` method should accept :class:`bytes` (a disk file
+ opened in binary mode, a :class:`~io.BytesIO` object, etc)
+ """
def __init__(self, file, **kwargs):
- self._configure(kwargs)
+ super().__init__(**kwargs)
self.file = file
def export_item(self, item):
@@ -287,7 +280,7 @@ class MarshalItemExporter(BaseItemExporter):
class PprintItemExporter(BaseItemExporter):
def __init__(self, file, **kwargs):
- self._configure(kwargs)
+ super().__init__(**kwargs)
self.file = file
def export_item(self, item):
@@ -296,10 +289,13 @@ class PprintItemExporter(BaseItemExporter):
class PythonItemExporter(BaseItemExporter):
- """The idea behind this exporter is to have a mechanism to serialize items
- to built-in python types so any serialization library (like
- json, msgpack, binc, etc) can be used on top of it. Its main goal is to
- seamless support what BaseItemExporter does plus nested items.
+ """This is a base class for item exporters that extends
+ :class:`BaseItemExporter` with support for nested items.
+
+ It serializes items to built-in Python types, so that any serialization
+ library (e.g. :mod:`json` or msgpack_) can be used on top of it.
+
+ .. _msgpack: https://pypi.org/project/msgpack/
"""
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
@@ -323,12 +319,12 @@ class PythonItemExporter(BaseItemExporter):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
- if isinstance(value, (six.text_type, bytes)):
+ if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return value
def _serialize_dict(self, value):
- for key, val in six.iteritems(value):
+ for key, val in value.items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(val)
diff --git a/scrapy/extension.py b/scrapy/extension.py
index e39e456fa..050b87e5f 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
+
class ExtensionManager(MiddlewareManager):
component_name = 'extension'
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 9ccf356ec..afb2ed049 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -54,9 +54,9 @@ class CloseSpider(object):
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
def spider_opened(self, spider):
- self.task = reactor.callLater(self.close_on['timeout'], \
- self.crawler.engine.close_spider, spider, \
- reason='closespider_timeout')
+ self.task = reactor.callLater(self.close_on['timeout'],
+ self.crawler.engine.close_spider, spider,
+ reason='closespider_timeout')
def item_scraped(self, item, spider):
self.counter['itemcount'] += 1
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 3d9a307b7..20adfbe4b 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -1,14 +1,16 @@
"""
Extension for collecting core stats like items scraped and start/finish times
"""
-import datetime
+from datetime import datetime
from scrapy import signals
+
class CoreStats(object):
def __init__(self, stats):
self.stats = stats
+ self.start_time = None
@classmethod
def from_crawler(cls, crawler):
@@ -21,10 +23,15 @@ class CoreStats(object):
return o
def spider_opened(self, spider):
- self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
+ self.start_time = datetime.utcnow()
+ self.stats.set_value('start_time', self.start_time, spider=spider)
def spider_closed(self, spider, reason):
- self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
+ finish_time = datetime.utcnow()
+ elapsed_time = finish_time - self.start_time
+ elapsed_time_seconds = elapsed_time.total_seconds()
+ self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
+ self.stats.set_value('finish_time', finish_time, spider=spider)
self.stats.set_value('finish_reason', reason, spider=spider)
def item_scraped(self, item, spider):
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 5f133fbde..f1b101780 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -7,21 +7,18 @@ See documentation in docs/topics/feed-exports.rst
import os
import sys
import logging
-import posixpath
from tempfile import NamedTemporaryFile
from datetime import datetime
-import six
-from six.moves.urllib.parse import urlparse
-from ftplib import FTP
+from urllib.parse import urlparse, unquote
from zope.interface import Interface, implementer
from twisted.internet import defer, threads
from w3lib.url import file_uri_to_path
from scrapy import signals
-from scrapy.utils.ftp import ftp_makedirs_cwd
+from scrapy.utils.ftp import ftp_store_file
from scrapy.exceptions import NotConfigured
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import without_none_values
from scrapy.utils.boto import is_botocore
@@ -65,7 +62,7 @@ class StdoutFeedStorage(object):
def __init__(self, uri, _stdout=None):
if not _stdout:
- _stdout = sys.stdout if six.PY2 else sys.stdout.buffer
+ _stdout = sys.stdout.buffer
self._stdout = _stdout
def open(self, spider):
@@ -93,14 +90,33 @@ class FileFeedStorage(object):
class S3FeedStorage(BlockingFeedStorage):
- def __init__(self, uri):
- from scrapy.conf import settings
+ def __init__(self, uri, access_key=None, secret_key=None, acl=None):
+ # BEGIN Backward compatibility for initialising without keys (and
+ # without using from_crawler)
+ no_defaults = access_key is None and secret_key is None
+ if no_defaults:
+ from scrapy.utils.project import get_project_settings
+ settings = get_project_settings()
+ if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
+ import warnings
+ from scrapy.exceptions import ScrapyDeprecationWarning
+ warnings.warn(
+ "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
+ "without AWS keys is deprecated. Please supply credentials or "
+ "use the `from_crawler()` constructor.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2
+ )
+ access_key = settings['AWS_ACCESS_KEY_ID']
+ secret_key = settings['AWS_SECRET_ACCESS_KEY']
+ # END Backward compatibility
u = urlparse(uri)
self.bucketname = u.hostname
- self.access_key = u.username or settings['AWS_ACCESS_KEY_ID']
- self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
+ self.access_key = u.username or access_key
+ self.secret_key = u.password or secret_key
self.is_botocore = is_botocore()
self.keyname = u.path[1:] # remove first "/"
+ self.acl = acl
if self.is_botocore:
import botocore.session
session = botocore.session.get_session()
@@ -111,38 +127,55 @@ class S3FeedStorage(BlockingFeedStorage):
import boto
self.connect_s3 = boto.connect_s3
+ @classmethod
+ def from_crawler(cls, crawler, uri):
+ return cls(
+ uri=uri,
+ access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
+ secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
+ acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
+ )
+
def _store_in_thread(self, file):
file.seek(0)
if self.is_botocore:
+ kwargs = {'ACL': self.acl} if self.acl else {}
self.s3_client.put_object(
- Bucket=self.bucketname, Key=self.keyname, Body=file)
+ Bucket=self.bucketname, Key=self.keyname, Body=file,
+ **kwargs)
else:
conn = self.connect_s3(self.access_key, self.secret_key)
bucket = conn.get_bucket(self.bucketname, validate=False)
key = bucket.new_key(self.keyname)
- key.set_contents_from_file(file)
+ kwargs = {'policy': self.acl} if self.acl else {}
+ key.set_contents_from_file(file, **kwargs)
key.close()
class FTPFeedStorage(BlockingFeedStorage):
- def __init__(self, uri):
+ def __init__(self, uri, use_active_mode=False):
u = urlparse(uri)
self.host = u.hostname
self.port = int(u.port or '21')
self.username = u.username
- self.password = u.password
+ self.password = unquote(u.password)
self.path = u.path
+ self.use_active_mode = use_active_mode
+
+ @classmethod
+ def from_crawler(cls, crawler, uri):
+ return cls(
+ uri=uri,
+ use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+ )
def _store_in_thread(self, file):
- file.seek(0)
- ftp = FTP()
- ftp.connect(self.host, self.port)
- ftp.login(self.username, self.password)
- dirname, filename = posixpath.split(self.path)
- ftp_makedirs_cwd(ftp, dirname)
- ftp.storbinary('STOR %s' % filename, file)
- ftp.quit()
+ ftp_store_file(
+ path=self.path, file=file, host=self.host,
+ port=self.port, username=self.username,
+ password=self.password, use_active_mode=self.use_active_mode
+ )
class SpiderSlot(object):
@@ -158,9 +191,9 @@ class FeedExporter(object):
def __init__(self, settings):
self.settings = settings
- self.urifmt = settings['FEED_URI']
- if not self.urifmt:
+ if not settings['FEED_URI']:
raise NotConfigured
+ self.urifmt = str(settings['FEED_URI'])
self.format = settings['FEED_FORMAT'].lower()
self.export_encoding = settings['FEED_EXPORT_ENCODING']
self.storages = self._load_components('FEED_STORAGES')
@@ -181,6 +214,7 @@ class FeedExporter(object):
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.settings)
+ o.crawler = crawler
crawler.signals.connect(o.open_spider, signals.spider_opened)
crawler.signals.connect(o.close_spider, signals.spider_closed)
crawler.signals.connect(o.item_scraped, signals.item_scraped)
@@ -200,7 +234,9 @@ class FeedExporter(object):
def close_spider(self, spider):
slot = self.slot
if not slot.itemcount and not self.store_empty:
- return
+ # We need to call slot.storage.store nonetheless to get the file
+ # properly closed.
+ return defer.maybeDeferred(slot.storage.store, slot.file)
if self._exporting:
slot.exporter.finish_exporting()
self._exporting = False
@@ -246,18 +282,24 @@ class FeedExporter(object):
try:
self._get_storage(uri)
return True
- except NotConfigured:
- logger.error("Disabled feed storage scheme: %(scheme)s",
- {'scheme': scheme})
+ except NotConfigured as e:
+ logger.error("Disabled feed storage scheme: %(scheme)s. "
+ "Reason: %(reason)s",
+ {'scheme': scheme, 'reason': str(e)})
else:
logger.error("Unknown feed storage scheme: %(scheme)s",
{'scheme': scheme})
+ def _get_instance(self, objcls, *args, **kwargs):
+ return create_instance(
+ objcls, self.settings, getattr(self, 'crawler', None),
+ *args, **kwargs)
+
def _get_exporter(self, *args, **kwargs):
- return self.exporters[self.format](*args, **kwargs)
+ return self._get_instance(self.exporters[self.format], *args, **kwargs)
def _get_storage(self, uri):
- return self.storages[urlparse(uri).scheme](uri)
+ return self._get_instance(self.storages[urlparse(uri).scheme], uri)
def _get_uri_params(self, spider):
params = {}
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 1b5e05b1b..91850683f 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,19 +1,20 @@
-from __future__ import print_function
-import os
import gzip
import logging
-from six.moves import cPickle as pickle
+import os
+import pickle
+from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from time import time
from weakref import WeakKeyDictionary
-from email.utils import mktime_tz, parsedate_tz
+
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
+
from scrapy.http import Headers, Response
from scrapy.responsetypes import responsetypes
-from scrapy.utils.request import request_fingerprint
-from scrapy.utils.project import data_path
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
+from scrapy.utils.project import data_path
+from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.request import request_fingerprint
logger = logging.getLogger(__name__)
@@ -31,7 +32,7 @@ class DummyPolicy(object):
def should_cache_response(self, response, request):
return response.status not in self.ignore_http_codes
- def is_cached_response_fresh(self, response, request):
+ def is_cached_response_fresh(self, cachedresponse, request):
return True
def is_cached_response_valid(self, cachedresponse, response, request):
@@ -70,7 +71,7 @@ class RFC2616Policy(object):
return True
def should_cache_response(self, response, request):
- # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
+ # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
# Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
# Status code 206 is not included because cache can not deal with partial contents
cc = self._parse_cachecontrol(response)
@@ -342,75 +343,6 @@ class FilesystemCacheStorage(object):
return pickle.load(f)
-class LeveldbCacheStorage(object):
-
- def __init__(self, settings):
- import leveldb
- self._leveldb = leveldb
- self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
- self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
- self.db = None
-
- def open_spider(self, spider):
- dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
- self.db = self._leveldb.LevelDB(dbpath)
-
- logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
-
- def close_spider(self, spider):
- # Do compactation each time to save space and also recreate files to
- # avoid them being removed in storages with timestamp-based autoremoval.
- self.db.CompactRange()
- del self.db
- garbage_collect()
-
- def retrieve_response(self, spider, request):
- data = self._read_data(spider, request)
- if data is None:
- return # not cached
- url = data['url']
- status = data['status']
- headers = Headers(data['headers'])
- body = data['body']
- respcls = responsetypes.from_args(headers=headers, url=url)
- response = respcls(url=url, headers=headers, status=status, body=body)
- return response
-
- def store_response(self, spider, request, response):
- key = self._request_key(request)
- data = {
- 'status': response.status,
- 'url': response.url,
- 'headers': dict(response.headers),
- 'body': response.body,
- }
- batch = self._leveldb.WriteBatch()
- batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
- batch.Put(key + b'_time', to_bytes(str(time())))
- self.db.Write(batch)
-
- def _read_data(self, spider, request):
- key = self._request_key(request)
- try:
- ts = self.db.Get(key + b'_time')
- except KeyError:
- return # not found or invalid entry
-
- if 0 < self.expiration_secs < time() - float(ts):
- return # expired
-
- try:
- data = self.db.Get(key + b'_data')
- except KeyError:
- return # invalid entry
- else:
- return pickle.loads(data)
-
- def _request_key(self, request):
- return to_bytes(request_fingerprint(request))
-
-
-
def parse_cachecontrol(header):
"""Parse Cache-Control header
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 263d8ce4c..892aa8a86 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
"""
import gc
-import six
from scrapy import signals
from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
def spider_closed(self, spider, reason):
gc.collect()
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 2220cbd8f..2c8e46914 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,10 +1,11 @@
import os
-from six.moves import cPickle as pickle
+import pickle
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.utils.job import job_dir
+
class SpiderState(object):
"""Store and load spider state during a scraping job"""
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index e78afa1fc..26b214ee2 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -6,13 +6,17 @@ See documentation in docs/topics/telnetconsole.rst
import pprint
import logging
+import traceback
+import binascii
+import os
from twisted.internet import protocol
try:
from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
TWISTED_CONCH_AVAILABLE = True
-except ImportError:
+except (ImportError, SyntaxError):
+ _TWISTED_CONCH_TRACEBACK = traceback.format_exc()
TWISTED_CONCH_AVAILABLE = False
from scrapy.exceptions import NotConfigured
@@ -20,6 +24,7 @@ from scrapy import signals
from scrapy.utils.trackref import print_live_refs
from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
+from scrapy.utils.decorators import defers
try:
import guppy
@@ -40,11 +45,20 @@ class TelnetConsole(protocol.ServerFactory):
if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
raise NotConfigured
if not TWISTED_CONCH_AVAILABLE:
- raise NotConfigured
+ raise NotConfigured(
+ 'TELNETCONSOLE_ENABLED setting is True but required twisted '
+ 'modules failed to import:\n' + _TWISTED_CONCH_TRACEBACK)
self.crawler = crawler
self.noisy = False
self.portrange = [int(x) for x in crawler.settings.getlist('TELNETCONSOLE_PORT')]
self.host = crawler.settings['TELNETCONSOLE_HOST']
+ self.username = crawler.settings['TELNETCONSOLE_USERNAME']
+ self.password = crawler.settings['TELNETCONSOLE_PASSWORD']
+
+ if not self.password:
+ self.password = binascii.hexlify(os.urandom(8)).decode('utf8')
+ logger.info('Telnet Password: %s', self.password)
+
self.crawler.signals.connect(self.start_listening, signals.engine_started)
self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
@@ -63,9 +77,25 @@ class TelnetConsole(protocol.ServerFactory):
self.port.stopListening()
def protocol(self):
- telnet_vars = self._get_telnet_vars()
- return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol,
- insults.ServerProtocol, manhole.Manhole, telnet_vars)
+ class Portal:
+ """An implementation of IPortal"""
+ @defers
+ def login(self_, credentials, mind, *interfaces):
+ if not (credentials.username == self.username.encode('utf8') and
+ credentials.checkPassword(self.password.encode('utf8'))):
+ raise ValueError("Invalid credentials")
+
+ protocol = telnet.TelnetBootstrapProtocol(
+ insults.ServerProtocol,
+ manhole.Manhole,
+ self._get_telnet_vars()
+ )
+ return (interfaces[0], protocol, lambda: None)
+
+ return telnet.TelnetTransport(
+ telnet.AuthenticatingTelnetProtocol,
+ Portal()
+ )
def _get_telnet_vars(self):
# Note: if you add entries here also update topics/telnetconsole.rst
@@ -81,8 +111,8 @@ class TelnetConsole(protocol.ServerFactory):
'p': pprint.pprint,
'prefs': print_live_refs,
'hpy': hpy,
- 'help': "This is Scrapy telnet console. For more info see: " \
- "https://doc.scrapy.org/en/latest/topics/telnetconsole.html",
+ 'help': "This is Scrapy telnet console. For more info see: "
+ "https://docs.scrapy.org/en/latest/topics/telnetconsole.html",
}
self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
return telnet_vars
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index f04a9d3e5..e6c58e1f1 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -10,6 +10,7 @@ from scrapy.http.headers import Headers
from scrapy.http.request import Request
from scrapy.http.request.form import FormRequest
from scrapy.http.request.rpc import XmlRpcRequest
+from scrapy.http.request.json_request import JsonRequest
from scrapy.http.response import Response
from scrapy.http.response.html import HtmlResponse
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index a1e95102e..0903fd4f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,9 +1,8 @@
import time
-from six.moves.http_cookiejar import (
- CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
-)
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_native_str
+from scrapy.utils.python import to_unicode
class CookieJar(object):
@@ -58,8 +57,8 @@ class CookieJar(object):
def clear_session_cookies(self, *args, **kwargs):
return self.jar.clear_session_cookies(*args, **kwargs)
- def clear(self):
- return self.jar.clear()
+ def clear(self, domain=None, path=None, name=None):
+ return self.jar.clear(domain, path, name)
def __iter__(self):
return iter(self.jar)
@@ -165,13 +164,13 @@ class WrappedRequest(object):
return name in self.request.headers
def get_header(self, name, default=None):
- return to_native_str(self.request.headers.get(name, default),
- errors='replace')
+ return to_unicode(self.request.headers.get(name, default),
+ errors='replace')
def header_items(self):
return [
- (to_native_str(k, errors='replace'),
- [to_native_str(x, errors='replace') for x in v])
+ (to_unicode(k, errors='replace'),
+ [to_unicode(x, errors='replace') for x in v])
for k, v in self.request.headers.items()
]
@@ -189,7 +188,7 @@ class WrappedResponse(object):
# python3 cookiejars calls get_all
def get_all(self, name, default=None):
- return [to_native_str(v, errors='replace')
+ return [to_unicode(v, errors='replace')
for v in self.response.headers.getlist(name)]
# python2 cookiejars calls getheaders
getheaders = get_all
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 62507eb19..dcaaeddfa 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,4 +1,3 @@
-import six
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@@ -19,7 +18,7 @@ class Headers(CaselessDict):
"""Normalize values to bytes"""
if value is None:
value = []
- elif isinstance(value, (six.text_type, bytes)):
+ elif isinstance(value, (str, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
@@ -29,10 +28,10 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
- elif isinstance(x, six.text_type):
+ elif isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
- return six.text_type(x).encode(self.encoding)
+ return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
@@ -68,9 +67,6 @@ class Headers(CaselessDict):
self[key] = lst
def items(self):
- return list(self.iteritems())
-
- def iteritems(self):
return ((k, self.getlist(k)) for k in self.keys())
def values(self):
@@ -91,5 +87,3 @@ class Headers(CaselessDict):
def __copy__(self):
return self.__class__(self)
copy = __copy__
-
-
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 13a92ffa0..6c536cb71 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,7 +4,6 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
-import six
from w3lib.url import safe_url_string
from scrapy.http.headers import Headers
@@ -12,13 +11,14 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
from scrapy.http.common import obsolete_setter
+from scrapy.utils.curl import curl_to_request_kwargs
class Request(object_ref):
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, flags=None):
+ dont_filter=False, errback=None, flags=None, cb_kwargs=None):
self._encoding = encoding # this one has to be set first
self.method = str(method).upper()
@@ -31,7 +31,6 @@ class Request(object_ref):
raise TypeError('callback must be a callable, got %s' % type(callback).__name__)
if errback is not None and not callable(errback):
raise TypeError('errback must be a callable, got %s' % type(errback).__name__)
- assert callback or not errback, "Cannot use errback without a callback"
self.callback = callback
self.errback = errback
@@ -40,8 +39,15 @@ class Request(object_ref):
self.dont_filter = dont_filter
self._meta = dict(meta) if meta else None
+ self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
self.flags = [] if flags is None else list(flags)
+ @property
+ def cb_kwargs(self):
+ if self._cb_kwargs is None:
+ self._cb_kwargs = {}
+ return self._cb_kwargs
+
@property
def meta(self):
if self._meta is None:
@@ -52,13 +58,13 @@ class Request(object_ref):
return self._url
def _set_url(self, url):
- if not isinstance(url, six.string_types):
+ if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
s = safe_url_string(url, self.encoding)
self._url = escape_ajax(s)
- if ':' not in self._url:
+ if ('://' not in self._url) and (not self._url.startswith('data:')):
raise ValueError('Missing scheme in request url: %s' % self._url)
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@@ -91,8 +97,39 @@ class Request(object_ref):
"""Create a new Request with the same attributes except for those
given new values.
"""
- for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta',
- 'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
+ for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
+ 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)
+
+ @classmethod
+ def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs):
+ """Create a Request object from a string containing a `cURL
+ <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
+ URL, the headers, the cookies and the body. It accepts the same
+ arguments as the :class:`Request` class, taking preference and
+ overriding the values of the same arguments contained in the cURL
+ command.
+
+ Unrecognized options are ignored by default. To raise an error when
+ finding unknown options call this method by passing
+ ``ignore_unknown_options=False``.
+
+ .. caution:: Using :meth:`from_curl` from :class:`~scrapy.http.Request`
+ subclasses, such as :class:`~scrapy.http.JSONRequest`, or
+ :class:`~scrapy.http.XmlRpcRequest`, as well as having
+ :ref:`downloader middlewares <topics-downloader-middleware>`
+ and
+ :ref:`spider middlewares <topics-spider-middleware>`
+ enabled, such as
+ :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`,
+ :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`,
+ or
+ :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
+ may modify the :class:`~scrapy.http.Request` object.
+
+ """
+ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
+ request_kwargs.update(kwargs)
+ return cls(**request_kwargs)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 95b38e990..af02c8484 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
-import six
-from six.moves.urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode
import lxml.html
from parsel.selector import create_root_node
@@ -18,6 +17,7 @@ from scrapy.utils.response import get_base_url
class FormRequest(Request):
+ valid_form_methods = ['GET', 'POST']
def __init__(self, *args, **kwargs):
formdata = kwargs.pop('formdata', None)
@@ -48,7 +48,13 @@ class FormRequest(Request):
form = _get_form(response, formname, formid, formnumber, formxpath)
formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
url = _get_form_url(form, kwargs.pop('url', None))
+
method = kwargs.pop('method', form.method)
+ if method is not None:
+ method = method.upper()
+ if method not in cls.valid_form_methods:
+ method = 'GET'
+
return cls(url=url, method=method, formdata=formdata, **kwargs)
@@ -97,8 +103,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
el = el.getparent()
if el is None:
break
- encoded = formxpath if six.PY3 else formxpath.encode('unicode_escape')
- raise ValueError('No <form> element found with %s' % encoded)
+ raise ValueError('No <form> element found with %s' % formxpath)
# If we get here, it means that either formname was None
# or invalid
@@ -114,10 +119,12 @@ def _get_form(response, formname, formid, formnumber, formxpath):
def _get_inputs(form, formdata, dont_click, clickdata, response):
try:
- formdata = dict(formdata or ())
+ formdata_keys = dict(formdata or ()).keys()
except (ValueError, TypeError):
raise ValueError('formdata should be a dict or iterable of tuples')
+ if not formdata:
+ formdata = ()
inputs = form.xpath('descendant::textarea'
'|descendant::select'
'|descendant::input[not(@type) or @type['
@@ -128,14 +135,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
"re": "http://exslt.org/regular-expressions"})
values = [(k, u'' if v is None else v)
for k, v in (_value(e) for e in inputs)
- if k and k not in formdata]
+ if k and k not in formdata_keys]
if not dont_click:
clickable = _get_clickable(clickdata, form)
if clickable and clickable[0] not in formdata and not clickable[0] is None:
values.append(clickable)
- values.extend((k, v) for k, v in formdata.items() if v is not None)
+ if isinstance(formdata, dict):
+ formdata = formdata.items()
+
+ values.extend((k, v) for k, v in formdata if v is not None)
return values
@@ -197,7 +207,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + \
- u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+ u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
new file mode 100644
index 000000000..f08b25280
--- /dev/null
+++ b/scrapy/http/request/json_request.py
@@ -0,0 +1,57 @@
+"""
+This module implements the JsonRequest class which is a more convenient class
+(than Request) to generate JSON Requests.
+
+See documentation in docs/topics/request-response.rst
+"""
+
+import copy
+import json
+import warnings
+
+from scrapy.http.request import Request
+from scrapy.utils.deprecate import create_deprecated_class
+
+
+class JsonRequest(Request):
+ def __init__(self, *args, **kwargs):
+ dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {}))
+ dumps_kwargs.setdefault('sort_keys', True)
+ self._dumps_kwargs = dumps_kwargs
+
+ body_passed = kwargs.get('body', None) is not None
+ data = kwargs.pop('data', None)
+ data_passed = data is not None
+
+ if body_passed and data_passed:
+ warnings.warn('Both body and data passed. data will be ignored')
+
+ elif not body_passed and data_passed:
+ kwargs['body'] = self._dumps(data)
+
+ if 'method' not in kwargs:
+ kwargs['method'] = 'POST'
+
+ super(JsonRequest, self).__init__(*args, **kwargs)
+ self.headers.setdefault('Content-Type', 'application/json')
+ self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
+
+ def replace(self, *args, **kwargs):
+ body_passed = kwargs.get('body', None) is not None
+ data = kwargs.pop('data', None)
+ data_passed = data is not None
+
+ if body_passed and data_passed:
+ warnings.warn('Both body and data passed. data will be ignored')
+
+ elif not body_passed and data_passed:
+ kwargs['body'] = self._dumps(data)
+
+ return super(JsonRequest, self).replace(*args, **kwargs)
+
+ def _dumps(self, data):
+ """Convert to JSON """
+ return json.dumps(data, **self._dumps_kwargs)
+
+
+JSONRequest = create_deprecated_class("JSONRequest", JsonRequest)
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index bd09f7534..811d3ad6b 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
-from six.moves import xmlrpc_client as xmlrpclib
+import xmlrpc.client as xmlrpclib
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 1974259b5..f92d0901c 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,14 +4,15 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
-from six.moves.urllib.parse import urljoin
+from typing import Generator
+from urllib.parse import urljoin
-from scrapy.http.request import Request
+from scrapy.exceptions import NotSupported
+from scrapy.http.common import obsolete_setter
from scrapy.http.headers import Headers
+from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref
-from scrapy.http.common import obsolete_setter
-from scrapy.exceptions import NotSupported
class Response(object_ref):
@@ -41,8 +42,8 @@ class Response(object_ref):
if isinstance(url, str):
self._url = url
else:
- raise TypeError('%s url must be str, got %s:' % (type(self).__name__,
- type(url).__name__))
+ raise TypeError('%s url must be str, got %s:' %
+ (type(self).__name__, type(url).__name__))
url = property(_get_url, obsolete_setter(_set_url, 'url'))
@@ -88,7 +89,7 @@ class Response(object_ref):
@property
def text(self):
"""For subclasses of TextResponse, this will return the body
- as text (unicode object in Python 2 and str in Python 3)
+ as str
"""
raise AttributeError("Response content isn't text")
@@ -106,15 +107,15 @@ class Response(object_ref):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None):
+ dont_filter=False, errback=None, cb_kwargs=None):
# type: (...) -> Request
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
-
- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
+
+ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
@@ -123,13 +124,51 @@ class Response(object_ref):
elif url is None:
raise ValueError("url can't be None")
url = self.urljoin(url)
- return Request(url, callback,
- method=method,
- headers=headers,
- body=body,
- cookies=cookies,
- meta=meta,
- encoding=encoding,
- priority=priority,
- dont_filter=dont_filter,
- errback=errback)
+ return Request(
+ url=url,
+ callback=callback,
+ method=method,
+ headers=headers,
+ body=body,
+ cookies=cookies,
+ meta=meta,
+ encoding=encoding,
+ priority=priority,
+ dont_filter=dont_filter,
+ errback=errback,
+ cb_kwargs=cb_kwargs,
+ )
+
+ def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
+ cookies=None, meta=None, encoding='utf-8', priority=0,
+ dont_filter=False, errback=None, cb_kwargs=None):
+ # type: (...) -> Generator[Request, None, None]
+ """
+ Return an iterable of :class:`~.Request` instances to follow all links
+ in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
+ but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
+ not only absolute URLs.
+
+ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
+ method which supports selectors in addition to absolute/relative URLs
+ and Link objects.
+ """
+ if not hasattr(urls, '__iter__'):
+ raise TypeError("'urls' argument must be an iterable")
+ return (
+ self.follow(
+ url=url,
+ callback=callback,
+ method=method,
+ headers=headers,
+ body=body,
+ cookies=cookies,
+ meta=meta,
+ encoding=encoding,
+ priority=priority,
+ dont_filter=dont_filter,
+ errback=errback,
+ cb_kwargs=cb_kwargs,
+ )
+ for url in urls
+ )
diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py
index bd3559fbb..7eed052c2 100644
--- a/scrapy/http/response/html.py
+++ b/scrapy/http/response/html.py
@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
from scrapy.http.response.text import TextResponse
+
class HtmlResponse(TextResponse):
pass
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 74a042f2c..09049c157 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,18 +5,19 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
-import six
-from six.moves.urllib.parse import urljoin
+from contextlib import suppress
+from typing import Generator
+from urllib.parse import urljoin
import parsel
-from w3lib.encoding import html_to_unicode, resolve_encoding, \
- html_body_declared_encoding, http_content_type_encoding
+from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
+ http_content_type_encoding, resolve_encoding)
from w3lib.html import strip_html5_whitespace
-from scrapy.http.request import Request
+from scrapy.http import Request
from scrapy.http.response import Response
+from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
-from scrapy.utils.python import memoizemethod_noargs, to_native_str
class TextResponse(Response):
@@ -31,20 +32,17 @@ class TextResponse(Response):
super(TextResponse, self).__init__(*args, **kwargs)
def _set_url(self, url):
- if isinstance(url, six.text_type):
- if six.PY2 and self.encoding is None:
- raise TypeError("Cannot convert unicode url - %s "
- "has no encoding" % type(self).__name__)
- self._url = to_native_str(url, self.encoding)
+ if isinstance(url, str):
+ self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
- if isinstance(body, six.text_type):
+ if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
- type(self).__name__)
+ type(self).__name__)
self._body = body.encode(self._encoding)
else:
super(TextResponse, self)._set_body(body)
@@ -84,14 +82,14 @@ class TextResponse(Response):
@memoizemethod_noargs
def _headers_encoding(self):
content_type = self.headers.get(b'Content-Type', b'')
- return http_content_type_encoding(to_native_str(content_type))
+ return http_content_type_encoding(to_unicode(content_type))
def _body_inferred_encoding(self):
if self._cached_benc is None:
- content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+ content_type = to_unicode(self.headers.get(b'Content-Type', b''))
benc, ubody = html_to_unicode(content_type, self.body,
- auto_detect_fun=self._auto_detect_fun,
- default_encoding=self._DEFAULT_ENCODING)
+ auto_detect_fun=self._auto_detect_fun,
+ default_encoding=self._DEFAULT_ENCODING)
self._cached_benc = benc
self._cached_ubody = ubody
return self._cached_benc
@@ -123,21 +121,22 @@ class TextResponse(Response):
def follow(self, url, callback=None, method='GET', headers=None, body=None,
cookies=None, meta=None, encoding=None, priority=0,
- dont_filter=False, errback=None):
+ dont_filter=False, errback=None, cb_kwargs=None):
# type: (...) -> Request
"""
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
-
- * a relative URL;
- * a scrapy.link.Link object (e.g. a link extractor result);
- * an attribute Selector (not SelectorList) - e.g.
+
+ * a relative URL
+ * a :class:`~scrapy.link.Link` object, e.g. the result of
+ :ref:`topics-link-extractors`
+ * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
+ ``response.css('a.my_link')[0]``
+ * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
``response.css('a::attr(href)')[0]`` or
- ``response.xpath('//img/@src')[0]``.
- * a Selector for ``<a>`` or ``<link>`` element, e.g.
- ``response.css('a.my_link')[0]``.
-
+ ``response.xpath('//img/@src')[0]``
+
See :ref:`response-follow-example` for usage examples.
"""
if isinstance(url, parsel.Selector):
@@ -145,7 +144,9 @@ class TextResponse(Response):
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
- return super(TextResponse, self).follow(url, callback,
+ return super(TextResponse, self).follow(
+ url=url,
+ callback=callback,
method=method,
headers=headers,
body=body,
@@ -154,22 +155,86 @@ class TextResponse(Response):
encoding=encoding,
priority=priority,
dont_filter=dont_filter,
- errback=errback
+ errback=errback,
+ cb_kwargs=cb_kwargs,
)
+ def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
+ cookies=None, meta=None, encoding=None, priority=0,
+ dont_filter=False, errback=None, cb_kwargs=None,
+ css=None, xpath=None):
+ # type: (...) -> Generator[Request, None, None]
+ """
+ A generator that produces :class:`~.Request` instances to follow all
+ links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
+ ``__init__`` method, except that each ``urls`` element does not need to be
+ an absolute URL, it can be any of the following:
+
+ * a relative URL
+ * a :class:`~scrapy.link.Link` object, e.g. the result of
+ :ref:`topics-link-extractors`
+ * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
+ ``response.css('a.my_link')[0]``
+ * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
+ ``response.css('a::attr(href)')[0]`` or
+ ``response.xpath('//img/@src')[0]``
+
+ In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
+ within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
+
+ Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or
+ using the ``css`` or ``xpath`` parameters, this method will not produce requests for
+ selectors from which links cannot be obtained (for instance, anchor tags without an
+ ``href`` attribute)
+ """
+ arg_count = len(list(filter(None, (urls, css, xpath))))
+ if arg_count != 1:
+ raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')
+ if not urls:
+ if css:
+ urls = self.css(css)
+ if xpath:
+ urls = self.xpath(xpath)
+ if isinstance(urls, parsel.SelectorList):
+ selectors = urls
+ urls = []
+ for sel in selectors:
+ with suppress(_InvalidSelector):
+ urls.append(_url_from_selector(sel))
+ return super(TextResponse, self).follow_all(
+ urls=urls,
+ callback=callback,
+ method=method,
+ headers=headers,
+ body=body,
+ cookies=cookies,
+ meta=meta,
+ encoding=encoding,
+ priority=priority,
+ dont_filter=dont_filter,
+ errback=errback,
+ cb_kwargs=cb_kwargs,
+ )
+
+
+class _InvalidSelector(ValueError):
+ """
+ Raised when a URL cannot be obtained from a Selector
+ """
+
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
- if isinstance(sel.root, six.string_types):
+ if isinstance(sel.root, str):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
- raise ValueError("Unsupported selector: %s" % sel)
+ raise _InvalidSelector("Unsupported selector: %s" % sel)
if sel.root.tag not in ('a', 'link'):
- raise ValueError("Only <a> and <link> elements are supported; got <%s>" %
- sel.root.tag)
+ raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" %
+ sel.root.tag)
href = sel.root.get('href')
if href is None:
- raise ValueError("<%s> element has no href attribute: %s" %
- (sel.root.tag, sel))
+ raise _InvalidSelector("<%s> element has no href attribute: %s" %
+ (sel.root.tag, sel))
return strip_html5_whitespace(href)
diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py
index 1df33fee5..abf474a2f 100644
--- a/scrapy/http/response/xml.py
+++ b/scrapy/http/response/xml.py
@@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst
from scrapy.http.response.text import TextResponse
+
class XmlResponse(TextResponse):
pass
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index eb93c6f7e..1896ec31e 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -1,5 +1,6 @@
from zope.interface import Interface
+
class ISpiderLoader(Interface):
def from_settings(settings):
@@ -15,8 +16,3 @@ class ISpiderLoader(Interface):
def find_by_request(request):
"""Return the list of spiders names that can handle the given request"""
-
-
-# ISpiderManager is deprecated, don't use it!
-# An alias is kept for backwards compatibility.
-ISpiderManager = ISpiderLoader
diff --git a/scrapy/item.py b/scrapy/item.py
index aa05e9c69..1d39b48b2 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,17 +4,30 @@ Scrapy Item
See documentation in docs/topics/item.rst
"""
-from pprint import pformat
-from collections import MutableMapping
-
from abc import ABCMeta
-import six
+from collections.abc import MutableMapping
+from copy import deepcopy
+from pprint import pformat
+from warnings import warn
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
class BaseItem(object_ref):
- """Base class for all scraped items."""
+ """Base class for all scraped items.
+
+ In Scrapy, an object is considered an *item* if it is an instance of either
+ :class:`BaseItem` or :class:`dict`. For example, when the output of a
+ spider callback is evaluated, only instances of :class:`BaseItem` or
+ :class:`dict` are passed to :ref:`item pipelines <topics-item-pipeline>`.
+
+ If you need instances of a custom class to be considered items by Scrapy,
+ you must inherit from either :class:`BaseItem` or :class:`dict`.
+
+ Unlike instances of :class:`dict`, instances of :class:`BaseItem` may be
+ :ref:`tracked <topics-leaks-trackrefs>` to debug memory leaks.
+ """
pass
@@ -23,6 +36,10 @@ class Field(dict):
class ItemMeta(ABCMeta):
+ """Metaclass_ of :class:`Item` that handles field definitions.
+
+ .. _metaclass: https://realpython.com/python-metaclasses
+ """
def __new__(mcs, class_name, bases, attrs):
classcell = attrs.pop('__classcell__', None)
@@ -49,10 +66,17 @@ class DictItem(MutableMapping, BaseItem):
fields = {}
+ def __new__(cls, *args, **kwargs):
+ if issubclass(cls, DictItem) and not issubclass(cls, Item):
+ warn('scrapy.item.DictItem is deprecated, please use '
+ 'scrapy.item.Item instead',
+ ScrapyDeprecationWarning, stacklevel=2)
+ return super(DictItem, cls).__new__(cls, *args, **kwargs)
+
def __init__(self, *args, **kwargs):
self._values = {}
if args or kwargs: # avoid creating dict for most common case
- for k, v in six.iteritems(dict(*args, **kwargs)):
+ for k, v in dict(*args, **kwargs).items():
self[k] = v
def __getitem__(self, key):
@@ -96,7 +120,13 @@ class DictItem(MutableMapping, BaseItem):
def copy(self):
return self.__class__(self)
+ def deepcopy(self):
+ """Return a `deep copy`_ of this item.
-@six.add_metaclass(ItemMeta)
-class Item(DictItem):
+ .. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy
+ """
+ return deepcopy(self)
+
+
+class Item(DictItem, metaclass=ItemMeta):
pass
diff --git a/scrapy/link.py b/scrapy/link.py
index 2c8301680..a809c5ca4 100644
--- a/scrapy/link.py
+++ b/scrapy/link.py
@@ -4,10 +4,6 @@ This module defines the Link object used in Link extractors.
For actual link extractors implementation see scrapy.linkextractors, or
its documentation in: docs/topics/link-extractors.rst
"""
-import warnings
-import six
-
-from scrapy.utils.python import to_bytes
class Link(object):
@@ -17,13 +13,8 @@ class Link(object):
def __init__(self, url, text='', fragment='', nofollow=False):
if not isinstance(url, str):
- if six.PY2:
- warnings.warn("Link urls must be str objects. "
- "Assuming utf-8 encoding (which could be wrong)")
- url = to_bytes(url, encoding='utf8')
- else:
- got = url.__class__.__name__
- raise TypeError("Link urls must be str objects, got %s" % got)
+ got = url.__class__.__name__
+ raise TypeError("Link urls must be str objects, got %s" % got)
self.url = url
self.text = text
self.fragment = fragment
@@ -39,4 +30,3 @@ class Link(object):
def __repr__(self):
return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
(self.url, self.text, self.fragment, self.nofollow)
-
diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py
deleted file mode 100644
index b744aff8e..000000000
--- a/scrapy/linkextractor.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.linkextractor` is deprecated, "
- "use `scrapy.linkextractors` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.linkextractors import *
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index c3c79cf25..bdeab3a75 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -6,11 +6,13 @@ This package contains a collection of Link Extractors.
For more info see docs/topics/link-extractors.rst
"""
import re
+from urllib.parse import urlparse
+from warnings import warn
-from six.moves.urllib.parse import urlparse
from parsel.csstranslator import HTMLTranslator
from w3lib.url import canonicalize_url
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.url import (
url_is_from_any_domain, url_has_any_extension,
@@ -19,37 +21,49 @@ from scrapy.utils.url import (
# common file extensions that are not followed if they occur in links
IGNORED_EXTENSIONS = [
+ # archives
+ '7z', '7zip', 'bz2', 'rar', 'tar', 'tar.gz', 'xz', 'zip',
+
# images
'mng', 'pct', 'bmp', 'gif', 'jpg', 'jpeg', 'png', 'pst', 'psp', 'tif',
- 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg',
+ 'tiff', 'ai', 'drw', 'dxf', 'eps', 'ps', 'svg', 'cdr', 'ico',
# audio
'mp3', 'wma', 'ogg', 'wav', 'ra', 'aac', 'mid', 'au', 'aiff',
# video
'3gp', 'asf', 'asx', 'avi', 'mov', 'mp4', 'mpg', 'qt', 'rm', 'swf', 'wmv',
- 'm4a', 'm4v', 'flv',
+ 'm4a', 'm4v', 'flv', 'webm',
# office suites
'xls', 'xlsx', 'ppt', 'pptx', 'pps', 'doc', 'docx', 'odt', 'ods', 'odg',
'odp',
# other
- 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar',
+ 'css', 'pdf', 'exe', 'bin', 'rss', 'dmg', 'iso', 'apk'
]
_re_type = type(re.compile("", 0))
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
-_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}
+_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
class FilteringLinkExtractor(object):
_csstranslator = HTMLTranslator()
+ def __new__(cls, *args, **kwargs):
+ from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
+ if (issubclass(cls, FilteringLinkExtractor) and
+ not issubclass(cls, LxmlLinkExtractor)):
+ warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
+ 'please use scrapy.linkextractors.LinkExtractor instead',
+ ScrapyDeprecationWarning, stacklevel=2)
+ return super(FilteringLinkExtractor, cls).__new__(cls)
+
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
- restrict_xpaths, canonicalize, deny_extensions, restrict_css):
+ restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
self.link_extractor = link_extractor
@@ -69,6 +83,8 @@ class FilteringLinkExtractor(object):
if deny_extensions is None:
deny_extensions = IGNORED_EXTENSIONS
self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)}
+ self.restrict_text = [x if isinstance(x, _re_type) else re.compile(x)
+ for x in arg_to_iter(restrict_text)]
def _link_allowed(self, link):
if not _is_valid_url(link.url):
@@ -84,6 +100,8 @@ class FilteringLinkExtractor(object):
return False
if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions):
return False
+ if self.restrict_text and not _matches(link.text, self.restrict_text):
+ return False
return True
def matches(self, url):
@@ -110,4 +128,4 @@ class FilteringLinkExtractor(object):
# Top-level imports
-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 27978a8a1..0425d4340 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -2,9 +2,8 @@
HTMLParser-based link extractor
"""
import warnings
-import six
-from six.moves.html_parser import HTMLParser
-from six.moves.urllib.parse import urljoin
+from html.parser import HTMLParser
+from urllib.parse import urljoin
from w3lib.url import safe_url_string
from w3lib.html import strip_html5_whitespace
@@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser):
ret = []
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in links:
- if isinstance(link.url, six.text_type):
+ if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index a7092f9b8..fdfa92370 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,8 +1,7 @@
"""
Link extractor based on lxml.html
"""
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
import lxml.etree as etree
from w3lib.html import strip_html5_whitespace
@@ -10,7 +9,7 @@ from w3lib.url import canonicalize_url
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_native_str
+from scrapy.utils.python import unique as unique_list, to_unicode
from scrapy.utils.response import get_base_url
from scrapy.linkextractors import FilteringLinkExtractor
@@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()")
def _nons(tag):
- if isinstance(tag, six.string_types):
+ if isinstance(tag, str):
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
return tag.split('}')[-1]
return tag
@@ -67,7 +66,7 @@ class LxmlParserLinkExtractor(object):
url = self.process_attr(attr_val)
if url is None:
continue
- url = to_native_str(url, encoding=response_encoding)
+ url = to_unicode(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(url, _collect_string_content(el) or u'',
@@ -97,7 +96,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=False,
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
- strip=True):
+ strip=True, restrict_text=None):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
@@ -111,11 +110,20 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
)
super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
- allow_domains=allow_domains, deny_domains=deny_domains,
- restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
- canonicalize=canonicalize, deny_extensions=deny_extensions)
+ allow_domains=allow_domains, deny_domains=deny_domains,
+ restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+ canonicalize=canonicalize, deny_extensions=deny_extensions,
+ restrict_text=restrict_text)
def extract_links(self, response):
+ """Returns a list of :class:`~scrapy.link.Link` objects from the
+ specified :class:`response <scrapy.http.Response>`.
+
+ Only links that match the settings passed to the ``__init__`` method of
+ the link extractor are returned.
+
+ Duplicate links are omitted.
+ """
base_url = get_base_url(response)
if self.restrict_xpaths:
docs = [subdoc
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index e689b4727..3f2557248 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -1,10 +1,11 @@
import re
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
from scrapy.link import Link
-from .sgml import SgmlLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor
+
linkre = re.compile(
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index f4ca4262a..2ba6bca45 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -1,9 +1,8 @@
"""
SGMLParser-based Link extractors
"""
-import six
-from six.moves.urllib.parse import urljoin
import warnings
+from urllib.parse import urljoin
from sgmllib import SGMLParser
from w3lib.url import safe_url_string, canonicalize_url
@@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
- if isinstance(link.url, six.text_type):
+ if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)
@@ -113,7 +112,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
process_value=None, deny_extensions=None, restrict_css=(),
- strip=True):
+ strip=True, restrict_text=()):
warnings.warn(
"SgmlLinkExtractor is deprecated and will be removed in future releases. "
"Please use scrapy.linkextractors.LinkExtractor",
@@ -127,13 +126,14 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
with warnings.catch_warnings():
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
- unique=unique, process_value=process_value, strip=strip,
- canonicalized=canonicalize)
+ unique=unique, process_value=process_value, strip=strip,
+ canonicalized=canonicalize)
super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
- allow_domains=allow_domains, deny_domains=deny_domains,
- restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
- canonicalize=canonicalize, deny_extensions=deny_extensions)
+ allow_domains=allow_domains, deny_domains=deny_domains,
+ restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
+ canonicalize=canonicalize, deny_extensions=deny_extensions,
+ restrict_text=restrict_text)
def extract_links(self, response):
base_url = None
@@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
base_url = get_base_url(response)
body = u''.join(f
for x in self.restrict_xpaths
- for f in response.xpath(x).extract()
+ for f in response.xpath(x).getall()
).encode(response.encoding, errors='xmlcharrefreplace')
else:
body = response.body
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index e73413318..7cf67e29e 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -1,20 +1,28 @@
-"""Item Loader
+"""
+Item Loader
See documentation in docs/topics/loaders.rst
-
"""
from collections import defaultdict
-import six
+from contextlib import suppress
from scrapy.item import Item
+from scrapy.loader.common import wrap_loader_context
+from scrapy.loader.processors import Identity
from scrapy.selector import Selector
-from scrapy.utils.decorators import deprecated
-from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.misc import arg_to_iter, extract_regex
from scrapy.utils.python import flatten
-from .common import wrap_loader_context
-from .processors import Identity
+
+def unbound_method(method):
+ """
+ Allow to use single-argument functions as input or output processors
+ (no need to define an unused first 'self' argument)
+ """
+ with suppress(AttributeError):
+ if '.' not in method.__qualname__:
+ return method.__func__
+ return method
class ItemLoader(object):
@@ -35,6 +43,9 @@ class ItemLoader(object):
self.parent = parent
self._local_item = context['item'] = item
self._local_values = defaultdict(list)
+ # values from initial item
+ for field_name, value in item.items():
+ self._values[field_name] += arg_to_iter(value)
@property
def _values(self):
@@ -71,7 +82,7 @@ class ItemLoader(object):
if value is None:
return
if not field_name:
- for k, v in six.iteritems(value):
+ for k, v in value.items():
self._add_value(k, v)
else:
self._add_value(field_name, value)
@@ -81,7 +92,7 @@ class ItemLoader(object):
if value is None:
return
if not field_name:
- for k, v in six.iteritems(value):
+ for k, v in value.items():
self._replace_value(k, v)
else:
self._replace_value(field_name, value)
@@ -105,8 +116,14 @@ class ItemLoader(object):
for proc in processors:
if value is None:
break
+ _proc = proc
proc = wrap_loader_context(proc, self.context)
- value = proc(value)
+ try:
+ value = proc(value)
+ except Exception as e:
+ raise ValueError("Error with processor %s value=%r error='%s: %s'" %
+ (_proc.__class__.__name__, value,
+ type(e).__name__, str(e)))
return value
def load_item(self):
@@ -124,8 +141,8 @@ class ItemLoader(object):
try:
return proc(self._values[field_name])
except Exception as e:
- raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
- (field_name, self._values[field_name], type(e).__name__, str(e)))
+ raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
+ (field_name, self._values[field_name], type(e).__name__, str(e)))
def get_collected_values(self, field_name):
return self._values[field_name]
@@ -133,21 +150,28 @@ class ItemLoader(object):
def get_input_processor(self, field_name):
proc = getattr(self, '%s_in' % field_name, None)
if not proc:
- proc = self._get_item_field_attr(field_name, 'input_processor', \
- self.default_input_processor)
- return proc
+ proc = self._get_item_field_attr(field_name, 'input_processor',
+ self.default_input_processor)
+ return unbound_method(proc)
def get_output_processor(self, field_name):
proc = getattr(self, '%s_out' % field_name, None)
if not proc:
- proc = self._get_item_field_attr(field_name, 'output_processor', \
- self.default_output_processor)
- return proc
+ proc = self._get_item_field_attr(field_name, 'output_processor',
+ self.default_output_processor)
+ return unbound_method(proc)
def _process_input_value(self, field_name, value):
proc = self.get_input_processor(field_name)
+ _proc = proc
proc = wrap_loader_context(proc, self.context)
- return proc(value)
+ try:
+ return proc(value)
+ except Exception as e:
+ raise ValueError(
+ "Error with input processor %s: field=%r value=%r "
+ "error='%s: %s'" % (_proc.__class__.__name__, field_name,
+ value, type(e).__name__, str(e)))
def _get_item_field_attr(self, field_name, key, default=None):
if isinstance(self.item, Item):
@@ -159,8 +183,8 @@ class ItemLoader(object):
def _check_selector_method(self):
if self.selector is None:
raise RuntimeError("To use XPath or CSS selectors, "
- "%s must be instantiated with a selector "
- "or a response" % self.__class__.__name__)
+ "%s must be instantiated with a selector "
+ "or a response" % self.__class__.__name__)
def add_xpath(self, field_name, xpath, *processors, **kw):
values = self._get_xpathvalues(xpath, **kw)
@@ -174,14 +198,10 @@ class ItemLoader(object):
values = self._get_xpathvalues(xpath, **kw)
return self.get_value(values, *processors, **kw)
- @deprecated(use_instead='._get_xpathvalues()')
- def _get_values(self, xpaths, **kw):
- return self._get_xpathvalues(xpaths, **kw)
-
def _get_xpathvalues(self, xpaths, **kw):
self._check_selector_method()
xpaths = arg_to_iter(xpaths)
- return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)
+ return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
def add_css(self, field_name, css, *processors, **kw):
values = self._get_cssvalues(css, **kw)
@@ -198,6 +218,4 @@ class ItemLoader(object):
def _get_cssvalues(self, csss, **kw):
self._check_selector_method()
csss = arg_to_iter(csss)
- return flatten(self.selector.css(css).extract() for css in csss)
-
-XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
+ return flatten(self.selector.css(css).getall() for css in csss)
diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py
index 916524947..42f8de636 100644
--- a/scrapy/loader/common.py
+++ b/scrapy/loader/common.py
@@ -3,6 +3,7 @@
from functools import partial
from scrapy.utils.python import get_func_args
+
def wrap_loader_context(function, context):
"""Wrap functions that receive loader_context to contain the context
"pre-loaded" and expose a interface that receives only one argument
diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py
index bf7c74bfe..02c625acc 100644
--- a/scrapy/loader/processors.py
+++ b/scrapy/loader/processors.py
@@ -3,10 +3,10 @@ This module provides some commonly used processors for Item Loaders.
See documentation in docs/topics/loaders.rst
"""
+from collections import ChainMap
from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.datatypes import MergeDict
-from .common import wrap_loader_context
+from scrapy.loader.common import wrap_loader_context
class MapCompose(object):
@@ -18,14 +18,20 @@ class MapCompose(object):
def __call__(self, value, loader_context=None):
values = arg_to_iter(value)
if loader_context:
- context = MergeDict(loader_context, self.default_loader_context)
+ context = ChainMap(loader_context, self.default_loader_context)
else:
context = self.default_loader_context
wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
for func in wrapped_funcs:
next_values = []
for v in values:
- next_values += arg_to_iter(func(v))
+ try:
+ next_values += arg_to_iter(func(v))
+ except Exception as e:
+ raise ValueError("Error in MapCompose with "
+ "%s value=%r error='%s: %s'" %
+ (str(func), value, type(e).__name__,
+ str(e)))
values = next_values
return values
@@ -39,14 +45,19 @@ class Compose(object):
def __call__(self, value, loader_context=None):
if loader_context:
- context = MergeDict(loader_context, self.default_loader_context)
+ context = ChainMap(loader_context, self.default_loader_context)
else:
context = self.default_loader_context
wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
for func in wrapped_funcs:
if value is None and self.stop_on_none:
break
- value = func(value)
+ try:
+ value = func(value)
+ except Exception as e:
+ raise ValueError("Error in Compose with "
+ "%s value=%r error='%s: %s'" %
+ (str(func), value, type(e).__name__, str(e)))
return value
diff --git a/scrapy/log.py b/scrapy/log.py
deleted file mode 100644
index 719fceaad..000000000
--- a/scrapy/log.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""
-This module is kept to provide a helpful warning about its removal.
-"""
-import logging
-import warnings
-
-from twisted.python.failure import Failure
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.log import failure_to_exc_info
-
-logger = logging.getLogger(__name__)
-
-warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
- "the builtin Python library for logging. Read the updated "
- "logging entry in the documentation to learn more.",
- ScrapyDeprecationWarning, stacklevel=2)
-
-
-# Imports and level_names variable kept for backwards-compatibility
-
-DEBUG = logging.DEBUG
-INFO = logging.INFO
-WARNING = logging.WARNING
-ERROR = logging.ERROR
-CRITICAL = logging.CRITICAL
-SILENT = CRITICAL + 1
-
-level_names = {
- logging.DEBUG: "DEBUG",
- logging.INFO: "INFO",
- logging.WARNING: "WARNING",
- logging.ERROR: "ERROR",
- logging.CRITICAL: "CRITICAL",
- SILENT: "SILENT",
-}
-
-
-def msg(message=None, _level=logging.INFO, **kw):
- warnings.warn('log.msg has been deprecated, create a python logger and '
- 'log through it instead',
- ScrapyDeprecationWarning, stacklevel=2)
-
- level = kw.pop('level', _level)
- message = kw.pop('format', message)
- # NOTE: logger.log doesn't handle well passing empty dictionaries with format
- # arguments because of some weird use-case:
- # https://hg.python.org/cpython/file/648dcafa7e5f/Lib/logging/__init__.py#l269
- logger.log(level, message, *[kw] if kw else [])
-
-
-def err(_stuff=None, _why=None, **kw):
- warnings.warn('log.err has been deprecated, create a python logger and '
- 'use its error method instead',
- ScrapyDeprecationWarning, stacklevel=2)
-
- level = kw.pop('level', logging.ERROR)
- failure = kw.pop('failure', _stuff) or Failure()
- message = kw.pop('why', _why) or failure.value
- logger.log(level, message, *[kw] if kw else [], exc_info=failure_to_exc_info(failure))
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 075a6d862..4e5963e99 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -8,30 +8,49 @@ from scrapy.utils.request import referer_str
SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
+ERRORMSG = u"'Error processing %(item)s'"
class LogFormatter(object):
"""Class for generating log messages for different actions.
- All methods must return a dictionary listing the parameters `level`, `msg`
- and `args` which are going to be used for constructing the log message when
- calling logging.log.
+ All methods must return a dictionary listing the parameters ``level``, ``msg``
+ and ``args`` which are going to be used for constructing the log message when
+ calling ``logging.log``.
Dictionary keys for the method outputs:
- * `level` should be the log level for that action, you can use those
- from the python logging library: logging.DEBUG, logging.INFO,
- logging.WARNING, logging.ERROR and logging.CRITICAL.
- * `msg` should be a string that can contain different formatting
- placeholders. This string, formatted with the provided `args`, is going
- to be the log message for that action.
+ * ``level`` is the log level for that action, you can use those from the
+ `python logging library <https://docs.python.org/3/library/logging.html>`_ :
+ ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR``
+ and ``logging.CRITICAL``.
+ * ``msg`` should be a string that can contain different formatting placeholders.
+ This string, formatted with the provided ``args``, is going to be the long message
+ for that action.
+ * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
+ The final log message is computed as ``msg % args``.
- * `args` should be a tuple or dict with the formatting placeholders for
- `msg`. The final log message is computed as output['msg'] %
- output['args'].
+ Users can define their own ``LogFormatter`` class if they want to customize how
+ each action is logged or if they want to omit it entirely. In order to omit
+ logging an action the method must return ``None``.
+
+ Here is an example on how to create a custom log formatter to lower the severity level of
+ the log message when an item is dropped from the pipeline::
+
+ class PoliteLogFormatter(logformatter.LogFormatter):
+ def dropped(self, item, exception, response, spider):
+ return {
+ 'level': logging.INFO, # lowering the level from logging.WARNING
+ 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+ 'args': {
+ 'exception': exception,
+ 'item': item,
+ }
+ }
"""
def crawled(self, request, response, spider):
+ """Logs a message when the crawler finds a webpage."""
request_flags = ' %s' % str(request.flags) if request.flags else ''
response_flags = ' %s' % str(response.flags) if response.flags else ''
return {
@@ -40,7 +59,7 @@ class LogFormatter(object):
'args': {
'status': response.status,
'request': request,
- 'request_flags' : request_flags,
+ 'request_flags': request_flags,
'referer': referer_str(request),
'response_flags': response_flags,
# backward compatibility with Scrapy logformatter below 1.4 version
@@ -49,6 +68,7 @@ class LogFormatter(object):
}
def scraped(self, item, response, spider):
+ """Logs a message when an item is scraped by a spider."""
if isinstance(response, Failure):
src = response.getErrorMessage()
else:
@@ -63,6 +83,7 @@ class LogFormatter(object):
}
def dropped(self, item, exception, response, spider):
+ """Logs a message when an item is dropped while it is passing through the item pipeline."""
return {
'level': logging.WARNING,
'msg': DROPPEDMSG,
@@ -72,6 +93,16 @@ class LogFormatter(object):
}
}
+ def error(self, item, exception, response, spider):
+ """Logs a message when an item causes an error while it is passing through the item pipeline."""
+ return {
+ 'level': logging.ERROR,
+ 'msg': ERRORMSG,
+ 'args': {
+ 'item': item,
+ }
+ }
+
@classmethod
def from_crawler(cls, crawler):
return cls()
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 5b944e1c4..9655b8114 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -4,29 +4,20 @@ Mail sending helpers
See documentation in docs/topics/email.rst
"""
import logging
-
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
-import six
-
+from email import encoders as Encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate
-from six.moves.email_mime_multipart import MIMEMultipart
-from six.moves.email_mime_text import MIMEText
-from six.moves.email_mime_base import MIMEBase
-if six.PY2:
- from email.MIMENonMultipart import MIMENonMultipart
- from email import Encoders
-else:
- from email.mime.nonmultipart import MIMENonMultipart
- from email import encoders as Encoders
+from io import BytesIO
from twisted.internet import defer, reactor, ssl
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
+
logger = logging.getLogger(__name__)
@@ -82,8 +73,7 @@ class MailSender(object):
part = MIMEBase(*mimetype.split('/'))
part.set_payload(f.read())
Encoders.encode_base64(part)
- part.add_header('Content-Disposition', 'attachment; filename="%s"' \
- % attach_name)
+ part.add_header('Content-Disposition', 'attachment', filename=attach_name)
msg.attach(part)
else:
msg.set_payload(body)
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index be36f977e..53fa435bb 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,9 +1,9 @@
-from collections import defaultdict
+from collections import defaultdict, deque
import logging
import pprint
from scrapy.exceptions import NotConfigured
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.defer import process_parallel, process_chain, process_chain_both
logger = logging.getLogger(__name__)
@@ -16,7 +16,7 @@ class MiddlewareManager(object):
def __init__(self, *middlewares):
self.middlewares = middlewares
- self.methods = defaultdict(list)
+ self.methods = defaultdict(deque)
for mw in middlewares:
self._add_middleware(mw)
@@ -32,12 +32,7 @@ class MiddlewareManager(object):
for clspath in mwlist:
try:
mwcls = load_object(clspath)
- if crawler and hasattr(mwcls, 'from_crawler'):
- mw = mwcls.from_crawler(crawler)
- elif hasattr(mwcls, 'from_settings'):
- mw = mwcls.from_settings(settings)
- else:
- mw = mwcls()
+ mw = create_instance(mwcls, settings, crawler)
middlewares.append(mw)
enabled.append(clspath)
except NotConfigured as e:
@@ -61,7 +56,7 @@ class MiddlewareManager(object):
if hasattr(mw, 'open_spider'):
self.methods['open_spider'].append(mw.open_spider)
if hasattr(mw, 'close_spider'):
- self.methods['close_spider'].insert(0, mw.close_spider)
+ self.methods['close_spider'].appendleft(mw.close_spider)
def _process_parallel(self, methodname, obj, *args):
return process_parallel(self.methods[methodname], obj, *args)
@@ -70,8 +65,8 @@ class MiddlewareManager(object):
return process_chain(self.methods[methodname], obj, *args)
def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
- return process_chain_both(self.methods[cb_methodname], \
- self.methods[eb_methodname], obj, *args)
+ return process_chain_both(self.methods[cb_methodname],
+ self.methods[eb_methodname], obj, *args)
def open_spider(self, spider):
return self._process_parallel('open_spider', spider)
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 2ef8786d0..b5725a8ee 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -6,6 +6,8 @@ See documentation in docs/item-pipeline.rst
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
+from scrapy.utils.defer import deferred_f_from_coro_f
+
class ItemPipelineManager(MiddlewareManager):
@@ -18,7 +20,7 @@ class ItemPipelineManager(MiddlewareManager):
def _add_middleware(self, pipe):
super(ItemPipelineManager, self)._add_middleware(pipe)
if hasattr(pipe, 'process_item'):
- self.methods['process_item'].append(pipe.process_item)
+ self.methods['process_item'].append(deferred_f_from_coro_f(pipe.process_item))
def process_item(self, item, spider):
return self._process_chain('process_item', item, spider)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 510cc23c7..9b7445755 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,19 +5,15 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
-import os
-import os.path
-import time
import logging
-from email.utils import parsedate_tz, mktime_tz
-from six.moves.urllib.parse import urlparse
+import mimetypes
+import os
+import time
from collections import defaultdict
-import six
-
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
+from email.utils import parsedate_tz, mktime_tz
+from ftplib import FTP
+from io import BytesIO
+from urllib.parse import urlparse
from twisted.internet import defer, threads
@@ -31,6 +27,8 @@ from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.ftp import ftp_store_file
+
logger = logging.getLogger(__name__)
@@ -156,14 +154,14 @@ class S3FilesStore(object):
Bucket=self.bucket,
Key=key_name,
Body=buf,
- Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+ Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
- for metakey, metavalue in six.iteritems(meta):
+ for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
@@ -189,9 +187,22 @@ class S3FilesStore(object):
'X-Amz-Grant-Read': 'GrantRead',
'X-Amz-Grant-Read-ACP': 'GrantReadACP',
'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
+ 'X-Amz-Object-Lock-Legal-Hold': 'ObjectLockLegalHoldStatus',
+ 'X-Amz-Object-Lock-Mode': 'ObjectLockMode',
+ 'X-Amz-Object-Lock-Retain-Until-Date': 'ObjectLockRetainUntilDate',
+ 'X-Amz-Request-Payer': 'RequestPayer',
+ 'X-Amz-Server-Side-Encryption': 'ServerSideEncryption',
+ 'X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id': 'SSEKMSKeyId',
+ 'X-Amz-Server-Side-Encryption-Context': 'SSEKMSEncryptionContext',
+ 'X-Amz-Server-Side-Encryption-Customer-Algorithm': 'SSECustomerAlgorithm',
+ 'X-Amz-Server-Side-Encryption-Customer-Key': 'SSECustomerKey',
+ 'X-Amz-Server-Side-Encryption-Customer-Key-Md5': 'SSECustomerKeyMD5',
+ 'X-Amz-Storage-Class': 'StorageClass',
+ 'X-Amz-Tagging': 'Tagging',
+ 'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
})
extra = {}
- for key, value in six.iteritems(headers):
+ for key, value in headers.items():
try:
kwarg = mapping[key]
except KeyError:
@@ -239,7 +250,7 @@ class GCSFilesStore(object):
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
- blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+ blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
@@ -248,6 +259,49 @@ class GCSFilesStore(object):
)
+class FTPFilesStore(object):
+
+ FTP_USERNAME = None
+ FTP_PASSWORD = None
+ USE_ACTIVE_MODE = None
+
+ def __init__(self, uri):
+ assert uri.startswith('ftp://')
+ u = urlparse(uri)
+ self.port = u.port
+ self.host = u.hostname
+ self.port = int(u.port or 21)
+ self.username = u.username or self.FTP_USERNAME
+ self.password = u.password or self.FTP_PASSWORD
+ self.basedir = u.path.rstrip('/')
+
+ def persist_file(self, path, buf, info, meta=None, headers=None):
+ path = '%s/%s' % (self.basedir, path)
+ return threads.deferToThread(
+ ftp_store_file, path=path, file=buf,
+ host=self.host, port=self.port, username=self.username,
+ password=self.password, use_active_mode=self.USE_ACTIVE_MODE
+ )
+
+ def stat_file(self, path, info):
+ def _stat_file(path):
+ try:
+ ftp = FTP()
+ ftp.connect(self.host, self.port)
+ ftp.login(self.username, self.password)
+ if self.USE_ACTIVE_MODE:
+ ftp.set_pasv(False)
+ file_path = "%s/%s" % (self.basedir, path)
+ last_modified = float(ftp.voidcmd("MDTM %s" % file_path)[4:].strip())
+ m = hashlib.md5()
+ ftp.retrbinary('RETR %s' % file_path, m.update)
+ return {'last_modified': last_modified, 'checksum': m.hexdigest()}
+ # The file doesn't exist
+ except Exception:
+ return {}
+ return threads.deferToThread(_stat_file, path)
+
+
class FilesPipeline(MediaPipeline):
"""Abstract pipeline that implement the file downloading
@@ -255,13 +309,13 @@ class FilesPipeline(MediaPipeline):
doing stat of the files and determining if file is new, uptodate or
expired.
- `new` files are those that pipeline never processed and needs to be
+ ``new`` files are those that pipeline never processed and needs to be
downloaded from supplier site the first time.
- `uptodate` files are the ones that the pipeline processed and are still
+ ``uptodate`` files are the ones that the pipeline processed and are still
valid files.
- `expired` files are those that pipeline already processed but the last
+ ``expired`` files are those that pipeline already processed but the last
modification was made long time ago, so a reprocessing is recommended to
refresh it in case of change.
@@ -274,6 +328,7 @@ class FilesPipeline(MediaPipeline):
'file': FSFilesStore,
's3': S3FilesStore,
'gs': GCSFilesStore,
+ 'ftp': FTPFilesStore
}
DEFAULT_FILES_URLS_FIELD = 'file_urls'
DEFAULT_FILES_RESULT_FIELD = 'files'
@@ -321,6 +376,11 @@ class FilesPipeline(MediaPipeline):
gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
gcs_store.POLICY = settings['FILES_STORE_GCS_ACL'] or None
+ ftp_store = cls.STORE_SCHEMES['ftp']
+ ftp_store.FTP_USERNAME = settings['FTP_USER']
+ ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
+ ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+
store_uri = settings['FILES_STORE']
return cls(store_uri, settings=settings)
@@ -458,33 +518,13 @@ class FilesPipeline(MediaPipeline):
return item
def file_path(self, request, response=None, info=None):
- ## start of deprecation warning block (can be removed in the future)
- def _warn():
- from scrapy.exceptions import ScrapyDeprecationWarning
- import warnings
- warnings.warn('FilesPipeline.file_key(url) method is deprecated, please use '
- 'file_path(request, response=None, info=None) instead',
- category=ScrapyDeprecationWarning, stacklevel=1)
-
- # check if called from file_key with url as first argument
- if not isinstance(request, Request):
- _warn()
- url = request
- else:
- url = request.url
-
- # detect if file_key() method has been overridden
- if not hasattr(self.file_key, '_base'):
- _warn()
- return self.file_key(url)
- ## end of deprecation warning block
-
- media_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation
- media_ext = os.path.splitext(url)[1] # change to request.url after deprecation
+ media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+ media_ext = os.path.splitext(request.url)[1]
+ # Handles empty and wild extensions by trying to guess the
+ # mime type then extension or default to empty string otherwise
+ if media_ext not in mimetypes.types_map:
+ media_ext = ''
+ media_type = mimetypes.guess_type(request.url)[0]
+ if media_type:
+ media_ext = mimetypes.guess_extension(media_type)
return 'full/%s%s' % (media_guid, media_ext)
-
- # deprecated
- def file_key(self, url):
- return self.file_path(url)
-
- file_key._base = True
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 95323c613..2e646379c 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -5,12 +5,7 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
-import six
-
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
+from io import BytesIO
from PIL import Image
@@ -89,12 +84,21 @@ class ImagesPipeline(FilesPipeline):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+ s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
+ s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
+ s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
+ s3store.AWS_VERIFY = settings['AWS_VERIFY']
s3store.POLICY = settings['IMAGES_STORE_S3_ACL']
gcs_store = cls.STORE_SCHEMES['gs']
gcs_store.GCS_PROJECT_ID = settings['GCS_PROJECT_ID']
gcs_store.POLICY = settings['IMAGES_STORE_GCS_ACL'] or None
+ ftp_store = cls.STORE_SCHEMES['ftp']
+ ftp_store.FTP_USERNAME = settings['FTP_USER']
+ ftp_store.FTP_PASSWORD = settings['FTP_PASSWORD']
+ ftp_store.USE_ACTIVE_MODE = settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+
store_uri = settings['IMAGES_STORE']
return cls(store_uri, settings=settings)
@@ -126,7 +130,7 @@ class ImagesPipeline(FilesPipeline):
image, buf = self.convert_image(orig_image)
yield path, image, buf
- for thumb_id, size in six.iteritems(self.thumbs):
+ for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
@@ -161,69 +165,9 @@ class ImagesPipeline(FilesPipeline):
return item
def file_path(self, request, response=None, info=None):
- ## start of deprecation warning block (can be removed in the future)
- def _warn():
- from scrapy.exceptions import ScrapyDeprecationWarning
- import warnings
- warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, '
- 'please use file_path(request, response=None, info=None) instead',
- category=ScrapyDeprecationWarning, stacklevel=1)
-
- # check if called from image_key or file_key with url as first argument
- if not isinstance(request, Request):
- _warn()
- url = request
- else:
- url = request.url
-
- # detect if file_key() or image_key() methods have been overridden
- if not hasattr(self.file_key, '_base'):
- _warn()
- return self.file_key(url)
- elif not hasattr(self.image_key, '_base'):
- _warn()
- return self.image_key(url)
- ## end of deprecation warning block
-
- image_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation
+ image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'full/%s.jpg' % (image_guid)
def thumb_path(self, request, thumb_id, response=None, info=None):
- ## start of deprecation warning block (can be removed in the future)
- def _warn():
- from scrapy.exceptions import ScrapyDeprecationWarning
- import warnings
- warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use '
- 'thumb_path(request, thumb_id, response=None, info=None) instead',
- category=ScrapyDeprecationWarning, stacklevel=1)
-
- # check if called from thumb_key with url as first argument
- if not isinstance(request, Request):
- _warn()
- url = request
- else:
- url = request.url
-
- # detect if thumb_key() method has been overridden
- if not hasattr(self.thumb_key, '_base'):
- _warn()
- return self.thumb_key(url, thumb_id)
- ## end of deprecation warning block
-
- thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation
+ thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid)
-
- # deprecated
- def file_key(self, url):
- return self.image_key(url)
- file_key._base = True
-
- # deprecated
- def image_key(self, url):
- return self.file_path(url)
- image_key._base = True
-
- # deprecated
- def thumb_key(self, url, thumb_id):
- return self.thumb_path(url, thumb_id)
- thumb_key._base = True
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 404bbf5bf..c174addf9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,9 +1,7 @@
-from __future__ import print_function
-
import functools
import logging
from collections import defaultdict
-from twisted.internet.defer import Deferred, DeferredList
+from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
from twisted.python.failure import Failure
from scrapy.settings import Settings
@@ -139,6 +137,30 @@ class MediaPipeline(object):
result.cleanFailure()
result.frames = []
result.stack = None
+
+ # This code fixes a memory leak by avoiding to keep references to
+ # the Request and Response objects on the Media Pipeline cache.
+ #
+ # Twisted inline callbacks pass return values using the function
+ # twisted.internet.defer.returnValue, which encapsulates the return
+ # value inside a _DefGen_Return base exception.
+ #
+ # What happens when the media_downloaded callback raises another
+ # exception, for example a FileException('download-error') when
+ # the Response status code is not 200 OK, is that it stores the
+ # _DefGen_Return exception on the FileException context.
+ #
+ # To avoid keeping references to the Response and therefore Request
+ # objects on the Media Pipeline cache, we should wipe the context of
+ # the exception encapsulated by the Twisted Failure when its a
+ # _DefGen_Return instance.
+ #
+ # This problem does not occur in Python 2.7 since we don't have
+ # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
+ context = getattr(result.value, '__context__', None)
+ if isinstance(context, _DefGen_Return):
+ setattr(result.value, '__context__', None)
+
info.downloading.remove(fp)
info.downloaded[fp] = result # cache result
for wad in info.waiting.pop(fp):
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
new file mode 100644
index 000000000..717ed4d27
--- /dev/null
+++ b/scrapy/pqueues.py
@@ -0,0 +1,190 @@
+import hashlib
+import logging
+from collections import namedtuple
+
+from queuelib import PriorityQueue
+
+from scrapy.utils.reqser import request_to_dict, request_from_dict
+
+
+logger = logging.getLogger(__name__)
+
+
+def _path_safe(text):
+ """
+ Return a filesystem-safe version of a string ``text``
+
+ >>> _path_safe('simple.org').startswith('simple.org')
+ True
+ >>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org')
+ True
+ >>> _path_safe('some@symbol?').startswith('some_symbol_')
+ True
+ """
+ pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_'
+ for c in text])
+ # as we replace some letters we can get collision for different slots
+ # add we add unique part
+ unique_slot = hashlib.md5(text.encode('utf8')).hexdigest()
+ return '-'.join([pathable_slot, unique_slot])
+
+
+class _Priority(namedtuple("_Priority", ["priority", "slot"])):
+ """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple
+ which can be used instead of int priorities in queues:
+
+ * they are ordered in the same way - order is still by priority value,
+ min(prios) works;
+ * str(p) representation is guaranteed to be different when slots
+ are different - this is important because str(p) is used to create
+ queue files on disk;
+ * they have readable str(p) representation which is safe
+ to use as a file name.
+ """
+ __slots__ = ()
+
+ def __str__(self):
+ return '%s_%s' % (self.priority, _path_safe(str(self.slot)))
+
+
+class _SlotPriorityQueues(object):
+ """ Container for multiple priority queues. """
+ def __init__(self, pqfactory, slot_startprios=None):
+ """
+ ``pqfactory`` is a factory for creating new PriorityQueues.
+ It must be a function which accepts a single optional ``startprios``
+ argument, with a list of priorities to create queues for.
+
+ ``slot_startprios`` is a ``{slot: startprios}`` dict.
+ """
+ self.pqfactory = pqfactory
+ self.pqueues = {} # slot -> priority queue
+ for slot, startprios in (slot_startprios or {}).items():
+ self.pqueues[slot] = self.pqfactory(startprios)
+
+ def pop_slot(self, slot):
+ """ Pop an object from a priority queue for this slot """
+ queue = self.pqueues[slot]
+ request = queue.pop()
+ if len(queue) == 0:
+ del self.pqueues[slot]
+ return request
+
+ def push_slot(self, slot, obj, priority):
+ """ Push an object to a priority queue for this slot """
+ if slot not in self.pqueues:
+ self.pqueues[slot] = self.pqfactory()
+ queue = self.pqueues[slot]
+ queue.push(obj, priority)
+
+ def close(self):
+ active = {slot: queue.close()
+ for slot, queue in self.pqueues.items()}
+ self.pqueues.clear()
+ return active
+
+ def __len__(self):
+ return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
+
+
+class ScrapyPriorityQueue(PriorityQueue):
+ """
+ PriorityQueue which works with scrapy.Request instances and
+ can optionally convert them to/from dicts before/after putting to a queue.
+ """
+ def __init__(self, crawler, qfactory, startprios=(), serialize=False):
+ super(ScrapyPriorityQueue, self).__init__(qfactory, startprios)
+ self.serialize = serialize
+ self.spider = crawler.spider
+
+ @classmethod
+ def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False):
+ return cls(crawler, qfactory, startprios, serialize)
+
+ def push(self, request, priority=0):
+ if self.serialize:
+ request = request_to_dict(request, self.spider)
+ super(ScrapyPriorityQueue, self).push(request, priority)
+
+ def pop(self):
+ request = super(ScrapyPriorityQueue, self).pop()
+ if request and self.serialize:
+ request = request_from_dict(request, self.spider)
+ return request
+
+
+class DownloaderInterface(object):
+
+ def __init__(self, crawler):
+ self.downloader = crawler.engine.downloader
+
+ def stats(self, possible_slots):
+ return [(self._active_downloads(slot), slot)
+ for slot in possible_slots]
+
+ def get_slot_key(self, request):
+ return self.downloader._get_slot_key(request, None)
+
+ def _active_downloads(self, slot):
+ """ Return a number of requests in a Downloader for a given slot """
+ if slot not in self.downloader.slots:
+ return 0
+ return len(self.downloader.slots[slot].active)
+
+
+class DownloaderAwarePriorityQueue(object):
+ """ PriorityQueue which takes Downlaoder activity in account:
+ domains (slots) with the least amount of active downloads are dequeued
+ first.
+ """
+
+ @classmethod
+ def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False):
+ return cls(crawler, qfactory, slot_startprios, serialize)
+
+ def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False):
+ if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
+ raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP'
+ % (self.__class__,))
+
+ if slot_startprios and not isinstance(slot_startprios, dict):
+ raise ValueError("DownloaderAwarePriorityQueue accepts "
+ "``slot_startprios`` as a dict; %r instance "
+ "is passed. Most likely, it means the state is"
+ "created by an incompatible priority queue. "
+ "Only a crawl started with the same priority "
+ "queue class can be resumed." %
+ slot_startprios.__class__)
+
+ slot_startprios = {
+ slot: [_Priority(p, slot) for p in startprios]
+ for slot, startprios in (slot_startprios or {}).items()}
+
+ def pqfactory(startprios=()):
+ return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize)
+ self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios)
+ self.serialize = serialize
+ self._downloader_interface = DownloaderInterface(crawler)
+
+ def pop(self):
+ stats = self._downloader_interface.stats(self._slot_pqueues.pqueues)
+
+ if not stats:
+ return
+
+ slot = min(stats)[1]
+ request = self._slot_pqueues.pop_slot(slot)
+ return request
+
+ def push(self, request, priority):
+ slot = self._downloader_interface.get_slot_key(request)
+ priority_slot = _Priority(priority=priority, slot=slot)
+ self._slot_pqueues.push_slot(slot, request, priority_slot)
+
+ def close(self):
+ active = self._slot_pqueues.close()
+ return {slot: [p.priority for p in startprios]
+ for slot, startprios in active.items()}
+
+ def __len__(self):
+ return len(self._slot_pqueues)
diff --git a/scrapy/project.py b/scrapy/project.py
deleted file mode 100644
index d8973a6c7..000000000
--- a/scrapy/project.py
+++ /dev/null
@@ -1,17 +0,0 @@
-
-"""
-Obsolete module, kept for giving a meaningful error message when trying to
-import.
-"""
-
-raise ImportError("""scrapy.project usage has become obsolete.
-
-If you want to get the Scrapy crawler from your extension, middleware or
-pipeline implement the `from_crawler` class method (or look up for extending
-components that have already done it, such as spiders).
-
-For example:
-
- @classmethod
- def from_crawler(cls, crawler):
- return cls(crawler)""")
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 0aaced7e4..554a3a14d 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,18 +1,37 @@
from twisted.internet import defer
from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
+from zope.interface.declarations import implementer, provider
from scrapy.utils.datatypes import LocalCache
-# TODO: cache misses
+# TODO: cache misses
dnscache = LocalCache(10000)
+
+@implementer(IResolverSimple)
class CachingThreadedResolver(ThreadedResolver):
+ """
+ Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests.
+ """
+
def __init__(self, reactor, cache_size, timeout):
super(CachingThreadedResolver, self).__init__(reactor)
dnscache.limit = cache_size
self.timeout = timeout
+ @classmethod
+ def from_crawler(cls, crawler, reactor):
+ if crawler.settings.getbool('DNSCACHE_ENABLED'):
+ cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+ else:
+ cache_size = 0
+ return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
+
+ def install_on_reactor(self,):
+ self.reactor.installResolver(self)
+
def getHostByName(self, name, timeout=None):
if name in dnscache:
return defer.succeed(dnscache[name])
@@ -29,3 +48,58 @@ class CachingThreadedResolver(ThreadedResolver):
def _cache_result(self, result, name):
dnscache[name] = result
return result
+
+
+@implementer(IHostnameResolver)
+class CachingHostnameResolver:
+ """
+ Experimental caching resolver. Resolves IPv4 and IPv6 addresses,
+ does not support setting a timeout value for DNS requests.
+ """
+
+ def __init__(self, reactor, cache_size):
+ self.reactor = reactor
+ self.original_resolver = reactor.nameResolver
+ dnscache.limit = cache_size
+
+ @classmethod
+ def from_crawler(cls, crawler, reactor):
+ if crawler.settings.getbool('DNSCACHE_ENABLED'):
+ cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+ else:
+ cache_size = 0
+ return cls(reactor, cache_size)
+
+ def install_on_reactor(self):
+ self.reactor.installNameResolver(self)
+
+ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
+ addressTypes=None, transportSemantics='TCP'):
+
+ @provider(IResolutionReceiver)
+ class CachingResolutionReceiver(resolutionReceiver):
+
+ def resolutionBegan(self, resolution):
+ super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+ self.resolution = resolution
+ self.resolved = False
+
+ def addressResolved(self, address):
+ super(CachingResolutionReceiver, self).addressResolved(address)
+ self.resolved = True
+
+ def resolutionComplete(self):
+ super(CachingResolutionReceiver, self).resolutionComplete()
+ if self.resolved:
+ dnscache[hostName] = self.resolution
+
+ try:
+ return dnscache[hostName]
+ except KeyError:
+ return self.original_resolver.resolveHostName(
+ CachingResolutionReceiver(),
+ hostName,
+ portNumber,
+ addressTypes,
+ transportSemantics
+ )
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4a2d5bf52..91d309147 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -2,15 +2,13 @@
This module implements a class which returns the appropriate Response class
based on different criteria.
"""
-from __future__ import absolute_import
from mimetypes import MimeTypes
from pkgutil import get_data
from io import StringIO
-import six
from scrapy.http import Response
from scrapy.utils.misc import load_object
-from scrapy.utils.python import binary_is_text, to_bytes, to_native_str
+from scrapy.utils.python import binary_is_text, to_bytes, to_unicode
class ResponseTypes(object):
@@ -37,7 +35,7 @@ class ResponseTypes(object):
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types').decode('utf8')
self.mimetypes.readfp(StringIO(mimedata))
- for mimetype, cls in six.iteritems(self.CLASSES):
+ for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
def from_mimetype(self, mimetype):
@@ -55,12 +53,12 @@ class ResponseTypes(object):
header """
if content_encoding:
return Response
- mimetype = to_native_str(content_type).split(';')[0].strip().lower()
+ mimetype = to_unicode(content_type).split(';')[0].strip().lower()
return self.from_mimetype(mimetype)
def from_content_disposition(self, content_disposition):
try:
- filename = to_native_str(content_disposition,
+ filename = to_unicode(content_disposition,
encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
filename = filename.strip('"\'')
return self.from_filename(filename)
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
new file mode 100644
index 000000000..0a9af3a62
--- /dev/null
+++ b/scrapy/robotstxt.py
@@ -0,0 +1,128 @@
+import sys
+import logging
+from abc import ABCMeta, abstractmethod
+
+from scrapy.utils.python import to_unicode
+
+
+logger = logging.getLogger(__name__)
+
+
+def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
+ try:
+ if to_native_str_type:
+ robotstxt_body = to_unicode(robotstxt_body)
+ else:
+ robotstxt_body = robotstxt_body.decode('utf-8')
+ except UnicodeDecodeError:
+ # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+ # Switch to 'allow all' state.
+ logger.warning("Failure while parsing robots.txt. "
+ "File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+ exc_info=sys.exc_info(),
+ extra={'spider': spider})
+ robotstxt_body = ''
+ return robotstxt_body
+
+
+class RobotParser(metaclass=ABCMeta):
+ @classmethod
+ @abstractmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ """Parse the content of a robots.txt_ file as bytes. This must be a class method.
+ It must return a new instance of the parser backend.
+
+ :param crawler: crawler which made the request
+ :type crawler: :class:`~scrapy.crawler.Crawler` instance
+
+ :param robotstxt_body: content of a robots.txt_ file.
+ :type robotstxt_body: bytes
+ """
+ pass
+
+ @abstractmethod
+ def allowed(self, url, user_agent):
+ """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
+
+ :param url: Absolute URL
+ :type url: string
+
+ :param user_agent: User agent
+ :type user_agent: string
+ """
+ pass
+
+
+class PythonRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from urllib.robotparser import RobotFileParser
+ self.spider = spider
+ robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
+ self.rp = RobotFileParser()
+ self.rp.parse(robotstxt_body.splitlines())
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ user_agent = to_unicode(user_agent)
+ url = to_unicode(url)
+ return self.rp.can_fetch(user_agent, url)
+
+
+class ReppyRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from reppy.robots import Robots
+ self.spider = spider
+ self.rp = Robots.parse('', robotstxt_body)
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ return self.rp.allowed(url, user_agent)
+
+
+class RerpRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from robotexclusionrulesparser import RobotExclusionRulesParser
+ self.spider = spider
+ self.rp = RobotExclusionRulesParser()
+ robotstxt_body = decode_robotstxt(robotstxt_body, spider)
+ self.rp.parse(robotstxt_body)
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ user_agent = to_unicode(user_agent)
+ url = to_unicode(url)
+ return self.rp.is_allowed(user_agent, url)
+
+
+class ProtegoRobotParser(RobotParser):
+ def __init__(self, robotstxt_body, spider):
+ from protego import Protego
+ self.spider = spider
+ robotstxt_body = decode_robotstxt(robotstxt_body, spider)
+ self.rp = Protego.parse(robotstxt_body)
+
+ @classmethod
+ def from_crawler(cls, crawler, robotstxt_body):
+ spider = None if not crawler else crawler.spider
+ o = cls(robotstxt_body, spider)
+ return o
+
+ def allowed(self, url, user_agent):
+ user_agent = to_unicode(user_agent)
+ url = to_unicode(url)
+ return self.rp.can_fetch(url, user_agent)
diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py
index bfbde4de9..a9240c1f6 100644
--- a/scrapy/selector/__init__.py
+++ b/scrapy/selector/__init__.py
@@ -1,5 +1,4 @@
"""
Selectors
"""
-from scrapy.selector.unified import *
-from scrapy.selector.lxmlsel import *
+from scrapy.selector.unified import * # noqa: F401
diff --git a/scrapy/selector/csstranslator.py b/scrapy/selector/csstranslator.py
deleted file mode 100644
index 8d7f034af..000000000
--- a/scrapy/selector/csstranslator.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
-from scrapy.utils.deprecate import create_deprecated_class
-
-
-ScrapyXPathExpr = create_deprecated_class(
- 'ScrapyXPathExpr', XPathExpr,
- new_class_path='parsel.csstranslator.XPathExpr')
-
-ScrapyGenericTranslator = create_deprecated_class(
- 'ScrapyGenericTranslator', GenericTranslator,
- new_class_path='parsel.csstranslator.GenericTranslator')
-
-ScrapyHTMLTranslator = create_deprecated_class(
- 'ScrapyHTMLTranslator', HTMLTranslator,
- new_class_path='parsel.csstranslator.HTMLTranslator')
diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py
deleted file mode 100644
index 070cb2306..000000000
--- a/scrapy/selector/lxmlsel.py
+++ /dev/null
@@ -1,50 +0,0 @@
-"""
-XPath selectors based on lxml
-"""
-from scrapy.utils.deprecate import create_deprecated_class
-from .unified import Selector, SelectorList
-
-
-__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector',
- 'XPathSelectorList']
-
-def _xpathselector_css(self, *a, **kw):
- raise RuntimeError('.css() method not available for %s, '
- 'instantiate scrapy.Selector '
- 'instead' % type(self).__name__)
-
-XPathSelector = create_deprecated_class(
- 'XPathSelector',
- Selector,
- {
- '__slots__': (),
- '_default_type': 'html',
- 'css': _xpathselector_css,
- },
- new_class_path='scrapy.Selector',
- old_class_path='scrapy.selector.XPathSelector',
-)
-
-XmlXPathSelector = create_deprecated_class(
- 'XmlXPathSelector',
- XPathSelector,
- clsdict={
- '__slots__': (),
- '_default_type': 'xml',
- },
- new_class_path='scrapy.Selector',
- old_class_path='scrapy.selector.XmlXPathSelector',
-)
-
-HtmlXPathSelector = create_deprecated_class(
- 'HtmlXPathSelector',
- XPathSelector,
- clsdict={
- '__slots__': (),
- '_default_type': 'html',
- },
- new_class_path='scrapy.Selector',
- old_class_path='scrapy.selector.HtmlXPathSelector',
-)
-
-XPathSelectorList = create_deprecated_class('XPathSelectorList', SelectorList)
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 64cb0232c..a08955dc9 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -2,13 +2,10 @@
XPath selectors based on lxml
"""
-import warnings
from parsel import Selector as _ParselSelector
from scrapy.utils.trackref import object_ref
from scrapy.utils.python import to_bytes
from scrapy.http import HtmlResponse, XmlResponse
-from scrapy.utils.decorators import deprecated
-from scrapy.exceptions import ScrapyDeprecationWarning
__all__ = ['Selector', 'SelectorList']
@@ -27,39 +24,53 @@ def _response_from_text(text, st):
class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
- @deprecated(use_instead='.extract()')
- def extract_unquoted(self):
- return [x.extract_unquoted() for x in self]
-
- @deprecated(use_instead='.xpath()')
- def x(self, xpath):
- return self.select(xpath)
-
- @deprecated(use_instead='.xpath()')
- def select(self, xpath):
- return self.xpath(xpath)
+ """
+ The :class:`SelectorList` class is a subclass of the builtin ``list``
+ class, which provides a few additional methods.
+ """
class Selector(_ParselSelector, object_ref):
+ """
+ An instance of :class:`Selector` is a wrapper over response to select
+ certain parts of its content.
+
+ ``response`` is an :class:`~scrapy.http.HtmlResponse` or an
+ :class:`~scrapy.http.XmlResponse` object that will be used for selecting
+ and extracting data.
+
+ ``text`` is a unicode string or utf-8 encoded text for cases when a
+ ``response`` isn't available. Using ``text`` and ``response`` together is
+ undefined behavior.
+
+ ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
+ or ``None`` (default).
+
+ If ``type`` is ``None``, the selector automatically chooses the best type
+ based on ``response`` type (see below), or defaults to ``"html"`` in case it
+ is used together with ``text``.
+
+ If ``type`` is ``None`` and a ``response`` is passed, the selector type is
+ inferred from the response type as follows:
+
+ * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
+ * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
+ * ``"html"`` for anything else
+
+ Otherwise, if ``type`` is set, the selector type will be forced and no
+ detection will occur.
+ """
__slots__ = ['response']
selectorlist_cls = SelectorList
- def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
+ def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
if not(response is None or text is None):
raise ValueError('%s.__init__() received both response and text'
% self.__class__.__name__)
st = _st(response, type or self._default_type)
- if _root is not None:
- warnings.warn("Argument `_root` is deprecated, use `root` instead",
- ScrapyDeprecationWarning, stacklevel=2)
- if root is None:
- root = _root
- else:
- warnings.warn("Ignoring deprecated `_root` argument, using provided `root`")
-
if text is not None:
response = _response_from_text(text, st)
@@ -69,18 +80,3 @@ class Selector(_ParselSelector, object_ref):
self.response = response
super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
-
- # Deprecated api
- @property
- def _root(self):
- warnings.warn("Attribute `_root` is deprecated, use `root` instead",
- ScrapyDeprecationWarning, stacklevel=2)
- return self.root
-
- @deprecated(use_instead='.xpath()')
- def select(self, xpath):
- return self.xpath(xpath)
-
- @deprecated(use_instead='.extract()')
- def extract_unquoted(self):
- return self.extract()
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 28446a372..b6133619c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,15 +1,10 @@
-import six
import json
import copy
-import warnings
-from collections import MutableMapping
+from collections.abc import MutableMapping
from importlib import import_module
from pprint import pformat
-from scrapy.utils.deprecate import create_deprecated_class
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-from . import default_settings
+from scrapy.settings import default_settings
SETTINGS_PRIORITIES = {
@@ -27,7 +22,7 @@ def get_settings_priority(priority):
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
numerical value, or directly returns a given numerical priority.
"""
- if isinstance(priority, six.string_types):
+ if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
@@ -177,7 +172,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or [])
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = value.split(',')
return list(value)
@@ -198,7 +193,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or {})
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = json.loads(value)
return dict(value)
@@ -288,7 +283,7 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(module, six.string_types):
+ if isinstance(module, str):
module = import_module(module)
for key in dir(module):
if key.isupper():
@@ -317,14 +312,14 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(values, six.string_types):
+ if isinstance(values, str):
values = json.loads(values)
if values is not None:
if isinstance(values, BaseSettings):
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, values.getpriority(name))
else:
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, priority)
def delete(self, name, priority='project'):
@@ -381,7 +376,7 @@ class BaseSettings(MutableMapping):
def _to_dict(self):
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
- for k, v in six.iteritems(self)}
+ for k, v in self.items()}
def copy_to_dict(self):
"""
@@ -405,30 +400,6 @@ class BaseSettings(MutableMapping):
else:
p.text(pformat(self.copy_to_dict()))
- @property
- def overrides(self):
- warnings.warn("`Settings.overrides` attribute is deprecated and won't "
- "be supported in Scrapy 0.26, use "
- "`Settings.set(name, value, priority='cmdline')` instead",
- category=ScrapyDeprecationWarning, stacklevel=2)
- try:
- o = self._overrides
- except AttributeError:
- self._overrides = o = _DictProxy(self, 'cmdline')
- return o
-
- @property
- def defaults(self):
- warnings.warn("`Settings.defaults` attribute is deprecated and won't "
- "be supported in Scrapy 0.26, use "
- "`Settings.set(name, value, priority='default')` instead",
- category=ScrapyDeprecationWarning, stacklevel=2)
- try:
- o = self._defaults
- except AttributeError:
- self._defaults = o = _DictProxy(self, 'default')
- return o
-
class _DictProxy(MutableMapping):
@@ -473,35 +444,12 @@ class Settings(BaseSettings):
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities
- for name, val in six.iteritems(self):
+ for name, val in self.items():
if isinstance(val, dict):
self.set(name, BaseSettings(val, 'default'), 'default')
self.update(values, priority)
-class CrawlerSettings(Settings):
-
- def __init__(self, settings_module=None, **kw):
- self.settings_module = settings_module
- Settings.__init__(self, **kw)
-
- def __getitem__(self, opt_name):
- if opt_name in self.overrides:
- return self.overrides[opt_name]
- if self.settings_module and hasattr(self.settings_module, opt_name):
- return getattr(self.settings_module, opt_name)
- if opt_name in self.defaults:
- return self.defaults[opt_name]
- return Settings.__getitem__(self, opt_name)
-
- def __str__(self):
- return "<CrawlerSettings module=%r>" % self.settings_module
-
-CrawlerSettings = create_deprecated_class(
- 'CrawlerSettings', CrawlerSettings,
- new_class_path='scrapy.settings.Settings')
-
-
def iter_default_settings():
"""Return the default settings as an iterator of (name, value) tuples"""
for name in dir(default_settings):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 36e17ef6b..c10dc1a1c 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -17,10 +17,10 @@ import sys
from importlib import import_module
from os.path import join, abspath, dirname
-import six
-
AJAXCRAWL_ENABLED = False
+ASYNCIO_REACTOR = False
+
AUTOTHROTTLE_ENABLED = False
AUTOTHROTTLE_DEBUG = False
AUTOTHROTTLE_MAX_DELAY = 60.0
@@ -55,11 +55,12 @@ DEFAULT_REQUEST_HEADERS = {
}
DEPTH_LIMIT = 0
-DEPTH_STATS = True
+DEPTH_STATS_VERBOSE = False
DEPTH_PRIORITY = 0
DNSCACHE_ENABLED = True
DNSCACHE_SIZE = 10000
+DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
DNS_TIMEOUT = 60
DOWNLOAD_DELAY = 0
@@ -85,8 +86,10 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
-DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
- # also allowing negotiation
+DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
+# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
+DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
DOWNLOADER_MIDDLEWARES = {}
@@ -158,6 +161,9 @@ FEED_EXPORTERS_BASE = {
}
FEED_EXPORT_INDENT = 0
+FEED_STORAGE_FTP_ACTIVE = False
+FEED_STORAGE_S3_ACL = ''
+
FILES_STORE_S3_ACL = 'private'
FILES_STORE_GCS_ACL = ''
@@ -174,7 +180,7 @@ HTTPCACHE_ALWAYS_STORE = False
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_IGNORE_SCHEMES = ['file']
HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
-HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
+HTTPCACHE_DBM_MODULE = 'dbm'
HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
HTTPCACHE_GZIP = False
@@ -219,6 +225,7 @@ MEMUSAGE_NOTIFY_MAIL = []
MEMUSAGE_WARNING_MB = 0
METAREFRESH_ENABLED = True
+METAREFRESH_IGNORE_TAGS = ['script', 'noscript']
METAREFRESH_MAXDELAY = 100
NEWSPIDER_MODULE = ''
@@ -236,15 +243,19 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
RETRY_PRIORITY_ADJUST = -1
ROBOTSTXT_OBEY = False
+ROBOTSTXT_PARSER = 'scrapy.robotstxt.ProtegoRobotParser'
+ROBOTSTXT_USER_AGENT = None
SCHEDULER = 'scrapy.core.scheduler.Scheduler'
SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
-SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
+SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'
+
+SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
SPIDER_LOADER_WARN_ONLY = False
@@ -277,10 +288,13 @@ USER_AGENT = 'Scrapy/%s (+https://scrapy.org)' % import_module('scrapy').__versi
TELNETCONSOLE_ENABLED = 1
TELNETCONSOLE_PORT = [6023, 6073]
TELNETCONSOLE_HOST = '127.0.0.1'
+TELNETCONSOLE_USERNAME = 'scrapy'
+TELNETCONSOLE_PASSWORD = None
SPIDER_CONTRACTS = {}
SPIDER_CONTRACTS_BASE = {
'scrapy.contracts.default.UrlContract': 1,
+ 'scrapy.contracts.default.CallbackKeywordArgumentsContract': 1,
'scrapy.contracts.default.ReturnsContract': 2,
'scrapy.contracts.default.ScrapesContract': 3,
}
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 80b625633..a23b04df9 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -3,13 +3,11 @@
See documentation in docs/topics/shell.rst
"""
-from __future__ import print_function
-
import os
import signal
import warnings
-from twisted.internet import reactor, threads, defer
+from twisted.internet import threads, defer
from twisted.python import threadable
from w3lib.url import any_to_uri
@@ -100,6 +98,7 @@ class Shell(object):
return spider
def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
+ from twisted.internet import reactor
if isinstance(request_or_url, Request):
request = request_or_url
else:
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 296d27ed8..481d97e9a 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
from pydispatch import dispatcher
from scrapy.utils import signal as _signal
@@ -46,16 +45,14 @@ class SignalManager(object):
def send_catch_log_deferred(self, signal, **kwargs):
"""
- Like :meth:`send_catch_log` but supports returning `deferreds`_ from
- signal handlers.
+ Like :meth:`send_catch_log` but supports returning
+ :class:`~twisted.internet.defer.Deferred` objects from signal handlers.
Returns a Deferred that gets fired once all signal handlers
deferreds were fired. Send a signal, catch exceptions and log them.
The keyword arguments are passed to the signal handlers (connected
through the :meth:`connect` method).
-
- .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
"""
kwargs.setdefault('sender', self.sender)
return _signal.send_catch_log_deferred(signal, **kwargs)
diff --git a/scrapy/signals.py b/scrapy/signals.py
index de0886fb6..6b9125302 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -13,12 +13,14 @@ spider_closed = object()
spider_error = object()
request_scheduled = object()
request_dropped = object()
+request_reached_downloader = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_dropped = object()
+item_error = object()
-# for backwards compatibility
+# for backward compatibility
stats_spider_opened = spider_opened
stats_spider_closing = spider_closed
stats_spider_closed = spider_closed
diff --git a/scrapy/spider.py b/scrapy/spider.py
deleted file mode 100644
index 56a5a0a0b..000000000
--- a/scrapy/spider.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.spider` is deprecated, "
- "use `scrapy.spiders` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.spiders import *
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 7478faa78..3beca4060 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
from collections import defaultdict
import traceback
import warnings
diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py
deleted file mode 100644
index 220257bb1..000000000
--- a/scrapy/spidermanager.py
+++ /dev/null
@@ -1,7 +0,0 @@
-"""
-Backwards compatibility shim. Use scrapy.spiderloader instead.
-"""
-from scrapy.spiderloader import SpiderLoader
-from scrapy.utils.deprecate import create_deprecated_class
-
-SpiderManager = create_deprecated_class('SpiderManager', SpiderLoader)
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index e2f039146..34a87f2df 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
class DepthMiddleware(object):
- def __init__(self, maxdepth, stats=None, verbose_stats=False, prio=1):
+ def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
self.maxdepth = maxdepth
self.stats = stats
self.verbose_stats = verbose_stats
@@ -41,7 +41,7 @@ class DepthMiddleware(object):
extra={'spider': spider}
)
return False
- elif self.stats:
+ else:
if self.verbose_stats:
self.stats.inc_value('request_depth_count/%s' % depth,
spider=spider)
@@ -50,7 +50,7 @@ class DepthMiddleware(object):
return True
# base case (depth=0)
- if self.stats and 'depth' not in response.meta:
+ if 'depth' not in response.meta:
response.meta['depth'] = 0
if self.verbose_stats:
self.stats.inc_value('request_depth_count/0', spider=spider)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 310166cad..232e96cbb 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -3,7 +3,6 @@ Offsite Spider Middleware
See documentation in docs/topics/spider-middleware.rst
"""
-
import re
import logging
import warnings
@@ -35,8 +34,9 @@ class OffsiteMiddleware(object):
domain = urlparse_cached(x).hostname
if domain and domain not in self.domains_seen:
self.domains_seen.add(domain)
- logger.debug("Filtered offsite request to %(domain)r: %(request)s",
- {'domain': domain, 'request': x}, extra={'spider': spider})
+ logger.debug(
+ "Filtered offsite request to %(domain)r: %(request)s",
+ {'domain': domain, 'request': x}, extra={'spider': spider})
self.stats.inc_value('offsite/domains', spider=spider)
self.stats.inc_value('offsite/filtered', spider=spider)
else:
@@ -52,13 +52,15 @@ class OffsiteMiddleware(object):
"""Override this method to implement a different offsite policy"""
allowed_domains = getattr(spider, 'allowed_domains', None)
if not allowed_domains:
- return re.compile('') # allow all by default
+ return re.compile('') # allow all by default
url_pattern = re.compile("^https?://.*$")
for domain in allowed_domains:
if url_pattern.match(domain):
- warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning)
-
- regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None)
+ message = ("allowed_domains accepts only domains, not URLs. "
+ "Ignoring URL entry %s in allowed_domains." % domain)
+ warnings.warn(message, URLWarning)
+ domains = [re.escape(d) for d in allowed_domains if d is not None]
+ regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
return re.compile(regex)
def spider_opened(self, spider):
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 1ddfb37f4..dce2b3598 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,16 +2,15 @@
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
-from six.moves.urllib.parse import urlparse
import warnings
+from urllib.parse import urlparse
from w3lib.url import safe_url_string
from scrapy.http import Request, Response
from scrapy.exceptions import NotConfigured
from scrapy import signals
-from scrapy.utils.python import to_native_str
-from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
from scrapy.utils.url import strip_url
@@ -322,7 +321,7 @@ class RefererMiddleware(object):
if isinstance(resp_or_url, Response):
policy_header = resp_or_url.headers.get('Referrer-Policy')
if policy_header is not None:
- policy_name = to_native_str(policy_header.decode('latin1'))
+ policy_name = to_unicode(policy_header.decode('latin1'))
if policy_name is None:
return self.default_policy()
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 80a36cf6d..4ddd34760 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -10,8 +10,6 @@ from scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.deprecate import create_deprecated_class
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.python import iflatten
@@ -55,15 +53,6 @@ class Spider(object_ref):
spider._set_crawler(crawler)
return spider
- def set_crawler(self, crawler):
- warnings.warn("set_crawler is deprecated, instantiate and bound the "
- "spider to this crawler with from_crawler method "
- "instead.",
- category=ScrapyDeprecationWarning, stacklevel=2)
- assert not hasattr(self, 'crawler'), "Spider already bounded to a " \
- "crawler"
- self._set_crawler(crawler)
-
def _set_crawler(self, crawler):
self.crawler = crawler
self.settings = crawler.settings
@@ -71,6 +60,11 @@ class Spider(object_ref):
def start_requests(self):
cls = self.__class__
+ if not self.start_urls and hasattr(self, 'start_url'):
+ raise AttributeError(
+ "Crawling could not start: 'start_urls' not found "
+ "or empty (but found 'start_url' attribute instead, "
+ "did you miss an 's'?)")
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
warnings.warn(
"Spider.make_requests_from_url method is deprecated; it "
@@ -117,23 +111,7 @@ class Spider(object_ref):
__repr__ = __str__
-BaseSpider = create_deprecated_class('BaseSpider', Spider)
-
-
-class ObsoleteClass(object):
- def __init__(self, message):
- self.message = message
-
- def __getattr__(self, name):
- raise AttributeError(self.message)
-
-spiders = ObsoleteClass(
- '"from scrapy.spider import spiders" no longer works - use '
- '"from scrapy.spiderloader import SpiderLoader" and instantiate '
- 'it with your project settings"'
-)
-
# Top-level imports
-from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider
-from scrapy.spiders.sitemap import SitemapSpider
+from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401
+from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401
+from scrapy.spiders.sitemap import SitemapSpider # noqa: F401
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e5ac72e18..a2c364c0e 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,29 +6,64 @@ See documentation in docs/topics/spiders.rst
"""
import copy
-import six
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
-from scrapy.utils.spider import iterate_spider_output
+from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
+from scrapy.utils.python import get_func_args
+from scrapy.utils.spider import iterate_spider_output
-def identity(x):
+def _identity(x):
return x
+def _identity_process_request(request, response):
+ return request
+
+
+def _get_method(method, spider):
+ if callable(method):
+ return method
+ elif isinstance(method, str):
+ return getattr(spider, method, None)
+
+
+_default_link_extractor = LinkExtractor()
+
+
class Rule(object):
- def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity):
- self.link_extractor = link_extractor
+ def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
+ process_links=None, process_request=None, errback=None):
+ self.link_extractor = link_extractor or _default_link_extractor
self.callback = callback
+ self.errback = errback
self.cb_kwargs = cb_kwargs or {}
- self.process_links = process_links
- self.process_request = process_request
- if follow is None:
- self.follow = False if callback else True
- else:
- self.follow = follow
+ self.process_links = process_links or _identity
+ self.process_request = process_request or _identity_process_request
+ self.process_request_argcount = None
+ self.follow = follow if follow is not None else not callback
+
+ def _compile(self, spider):
+ self.callback = _get_method(self.callback, spider)
+ self.errback = _get_method(self.errback, spider)
+ self.process_links = _get_method(self.process_links, spider)
+ self.process_request = _get_method(self.process_request, spider)
+ self.process_request_argcount = len(get_func_args(self.process_request))
+ if self.process_request_argcount == 1:
+ msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated'
+ warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
+
+ def _process_request(self, request, response):
+ """
+ Wrapper around the request processing function to maintain backward
+ compatibility with functions that do not take a Response object
+ """
+ args = [request] if self.process_request_argcount == 1 else [request, response]
+ return self.process_request(*args)
class CrawlSpider(Spider):
@@ -48,60 +83,59 @@ class CrawlSpider(Spider):
def process_results(self, response, results):
return results
- def _build_request(self, rule, link):
- r = Request(url=link.url, callback=self._response_downloaded)
- r.meta.update(rule=rule, link_text=link.text)
- return r
+ def _build_request(self, rule_index, link):
+ return Request(
+ url=link.url,
+ callback=self._callback,
+ errback=self._errback,
+ meta=dict(rule=rule_index, link_text=link.text),
+ )
def _requests_to_follow(self, response):
if not isinstance(response, HtmlResponse):
return
seen = set()
- for n, rule in enumerate(self._rules):
+ for rule_index, rule in enumerate(self._rules):
links = [lnk for lnk in rule.link_extractor.extract_links(response)
if lnk not in seen]
- if links and rule.process_links:
- links = rule.process_links(links)
- for link in links:
+ for link in rule.process_links(links):
seen.add(link)
- r = self._build_request(n, link)
- yield rule.process_request(r)
+ request = self._build_request(rule_index, link)
+ yield rule._process_request(request, response)
- def _response_downloaded(self, response):
+ def _callback(self, response):
rule = self._rules[response.meta['rule']]
return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
+ def _errback(self, failure):
+ rule = self._rules[failure.request.meta['rule']]
+ return self._handle_failure(failure, rule.errback)
+
def _parse_response(self, response, callback, cb_kwargs, follow=True):
if callback:
cb_res = callback(response, **cb_kwargs) or ()
cb_res = self.process_results(response, cb_res)
- for requests_or_item in iterate_spider_output(cb_res):
- yield requests_or_item
+ for request_or_item in iterate_spider_output(cb_res):
+ yield request_or_item
if follow and self._follow_links:
for request_or_item in self._requests_to_follow(response):
yield request_or_item
- def _compile_rules(self):
- def get_method(method):
- if callable(method):
- return method
- elif isinstance(method, six.string_types):
- return getattr(self, method, None)
+ def _handle_failure(self, failure, errback):
+ if errback:
+ results = errback(failure) or ()
+ for request_or_item in iterate_spider_output(results):
+ yield request_or_item
- self._rules = [copy.copy(r) for r in self.rules]
- for rule in self._rules:
- rule.callback = get_method(rule.callback)
- rule.process_links = get_method(rule.process_links)
- rule.process_request = get_method(rule.process_request)
+ def _compile_rules(self):
+ self._rules = []
+ for rule in self.rules:
+ self._rules.append(copy.copy(rule))
+ self._rules[-1]._compile(self)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
- spider._follow_links = crawler.settings.getbool(
- 'CRAWLSPIDER_FOLLOW_LINKS', True)
+ spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
return spider
-
- def set_crawler(self, crawler):
- super(CrawlSpider, self).set_crawler(crawler)
- self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 06e212e1c..c566f0236 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -100,8 +100,8 @@ class CSVFeedSpider(Spider):
and the file's headers.
"""
- delimiter = None # When this is None, python's csv module's default delimiter is used
- quotechar = None # When this is None, python's csv module's default quotechar is used
+ delimiter = None # When this is None, python's csv module's default delimiter is used
+ quotechar = None # When this is None, python's csv module's default quotechar is used
headers = None
def process_results(self, response, results):
@@ -133,4 +133,3 @@ class CSVFeedSpider(Spider):
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
response = self.adapt_response(response)
return self.parse_rows(response)
-
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index 2efb1a869..fd41133ea 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -29,4 +29,3 @@ class InitSpider(Spider):
spider
"""
return self.initialized()
-
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 0ee8ba5e7..d368c7108 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,6 +1,5 @@
import re
import logging
-import six
from scrapy.spiders import Spider
from scrapy.http import Request, XmlResponse
@@ -22,7 +21,7 @@ class SitemapSpider(Spider):
super(SitemapSpider, self).__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
- if isinstance(c, six.string_types):
+ if isinstance(c, str):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
@@ -31,6 +30,14 @@ class SitemapSpider(Spider):
for url in self.sitemap_urls:
yield Request(url, self._parse_sitemap)
+ def sitemap_filter(self, entries):
+ """This method can be used to filter sitemap entries by their
+ attributes, for example, you can filter locs with lastmod greater
+ than a given date (see docs).
+ """
+ for entry in entries:
+ yield entry
+
def _parse_sitemap(self, response):
if response.url.endswith('/robots.txt'):
for url in sitemap_urls_from_robots(response.text, base_url=response.url):
@@ -43,12 +50,14 @@ class SitemapSpider(Spider):
return
s = Sitemap(body)
+ it = self.sitemap_filter(s)
+
if s.type == 'sitemapindex':
- for loc in iterloc(s, self.sitemap_alternate_links):
+ for loc in iterloc(it, self.sitemap_alternate_links):
if any(x.search(loc) for x in self._follow):
yield Request(loc, callback=self._parse_sitemap)
elif s.type == 'urlset':
- for loc in iterloc(s, self.sitemap_alternate_links):
+ for loc in iterloc(it, self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield Request(loc, callback=c)
@@ -76,7 +85,7 @@ class SitemapSpider(Spider):
def regex(x):
- if isinstance(x, six.string_types):
+ if isinstance(x, str):
return re.compile(x)
return x
diff --git a/scrapy/squeue.py b/scrapy/squeue.py
deleted file mode 100644
index a4a3f4238..000000000
--- a/scrapy/squeue.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.squeue` is deprecated, "
- "use `scrapy.squeues` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.squeues import *
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index d2074a457..d5d3be67e 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,10 +3,11 @@ Scheduler queues
"""
import marshal
-from six.moves import cPickle as pickle
+import pickle
from queuelib import queue
+
def _serializable_queue(queue_class, serialize, deserialize):
class SerializableQueue(queue_class):
@@ -22,6 +23,7 @@ def _serializable_queue(queue_class, serialize, deserialize):
return SerializableQueue
+
def _pickle_serialize(obj):
try:
return pickle.dumps(obj, protocol=2)
@@ -31,13 +33,14 @@ def _pickle_serialize(obj):
except (pickle.PicklingError, AttributeError, TypeError) as e:
raise ValueError(str(e))
-PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
+
+PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
_pickle_serialize, pickle.loads)
-PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
+PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
_pickle_serialize, pickle.loads)
-MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
+MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
marshal.dumps, marshal.loads)
-MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
+MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
marshal.dumps, marshal.loads)
FifoMemoryQueue = queue.FifoMemoryQueue
LifoMemoryQueue = queue.LifoMemoryQueue
diff --git a/scrapy/stats.py b/scrapy/stats.py
deleted file mode 100644
index 710601430..000000000
--- a/scrapy/stats.py
+++ /dev/null
@@ -1,8 +0,0 @@
-
-"""
-Obsolete module, kept for giving a meaningful error message when trying to
-import.
-"""
-
-raise ImportError("scrapy.stats usage has become obsolete, use "
- "`crawler.stats` attribute instead")
diff --git a/scrapy/statscol.py b/scrapy/statscol.py
deleted file mode 100644
index b4ddcce28..000000000
--- a/scrapy/statscol.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.statscol` is deprecated, "
- "use `scrapy.statscollectors` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.statscollectors import *
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index 6da9ddcd2..f0bfaed34 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -80,5 +80,3 @@ class DummyStatsCollector(StatsCollector):
def min_value(self, key, value, spider=None):
pass
-
-
diff --git a/scrapy/telnet.py b/scrapy/telnet.py
deleted file mode 100644
index 70915f486..000000000
--- a/scrapy/telnet.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.telnet` is deprecated, "
- "use `scrapy.extensions.telnet` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.telnet import *
diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl
index 7d766f4fc..a12d08414 100644
--- a/scrapy/templates/project/module/items.py.tmpl
+++ b/scrapy/templates/project/module/items.py.tmpl
@@ -3,7 +3,7 @@
# Define here the models for your scraped items
#
# See documentation in:
-# https://doc.scrapy.org/en/latest/topics/items.html
+# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl
index c5b542bd6..97b5db2e1 100644
--- a/scrapy/templates/project/module/middlewares.py.tmpl
+++ b/scrapy/templates/project/module/middlewares.py.tmpl
@@ -3,7 +3,7 @@
# Define here the models for your spider middleware
#
# See documentation in:
-# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
@@ -39,7 +39,7 @@ class ${ProjectName}SpiderMiddleware(object):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
- # Should return either None or an iterable of Response, dict
+ # Should return either None or an iterable of Request, dict
# or Item objects.
pass
diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl
index e58dab089..fb641d447 100644
--- a/scrapy/templates/project/module/pipelines.py.tmpl
+++ b/scrapy/templates/project/module/pipelines.py.tmpl
@@ -3,7 +3,7 @@
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
-# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
class ${ProjectName}Pipeline(object):
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index a0557473e..cb220eafc 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -5,9 +5,9 @@
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
-# https://doc.scrapy.org/en/latest/topics/settings.html
-# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
-# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+# https://docs.scrapy.org/en/latest/topics/settings.html
+# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = '$project_name'
@@ -25,7 +25,7 @@ ROBOTSTXT_OBEY = True
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
-# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
@@ -45,31 +45,31 @@ ROBOTSTXT_OBEY = True
#}
# Enable or disable spider middlewares
-# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# '$project_name.middlewares.${ProjectName}SpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
-# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# '$project_name.middlewares.${ProjectName}DownloaderMiddleware': 543,
#}
# Enable or disable extensions
-# See https://doc.scrapy.org/en/latest/topics/extensions.html
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
-# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# '$project_name.pipelines.${ProjectName}Pipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
-# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
@@ -82,7 +82,7 @@ ROBOTSTXT_OBEY = True
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
-# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl
index 802cb88a1..878425125 100644
--- a/scrapy/templates/spiders/crawl.tmpl
+++ b/scrapy/templates/spiders/crawl.tmpl
@@ -14,8 +14,8 @@ class $classname(CrawlSpider):
)
def parse_item(self, response):
- i = {}
- #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
- #i['name'] = response.xpath('//div[@id="name"]').extract()
- #i['description'] = response.xpath('//div[@id="description"]').extract()
- return i
+ item = {}
+ #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
+ #item['name'] = response.xpath('//div[@id="name"]').get()
+ #item['description'] = response.xpath('//div[@id="description"]').get()
+ return item
diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl
index 7c2ff8850..863c9772f 100644
--- a/scrapy/templates/spiders/xmlfeed.tmpl
+++ b/scrapy/templates/spiders/xmlfeed.tmpl
@@ -10,8 +10,8 @@ class $classname(XMLFeedSpider):
itertag = 'item' # change it accordingly
def parse_node(self, response, selector):
- i = {}
- #i['url'] = selector.select('url').extract()
- #i['name'] = selector.select('name').extract()
- #i['description'] = selector.select('description').extract()
- return i
+ item = {}
+ #item['url'] = selector.select('url').get()
+ #item['name'] = selector.select('name').get()
+ #item['description'] = selector.select('description').get()
+ return item
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
new file mode 100644
index 000000000..917973de2
--- /dev/null
+++ b/scrapy/utils/asyncio.py
@@ -0,0 +1,17 @@
+import asyncio
+from contextlib import suppress
+
+from twisted.internet import asyncioreactor
+from twisted.internet.error import ReactorAlreadyInstalledError
+
+
+def install_asyncio_reactor():
+ """ Tries to install AsyncioSelectorReactor
+ """
+ with suppress(ReactorAlreadyInstalledError):
+ asyncioreactor.install(asyncio.get_event_loop())
+
+
+def is_asyncio_reactor_installed():
+ from twisted.internet import reactor
+ return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 5bbda6e27..cdbe21942 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -1,5 +1,6 @@
import random
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
+
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.internet import reactor
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 421ab2f7e..12321caa5 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,21 +1,11 @@
"""Boto/botocore helpers"""
-from __future__ import absolute_import
-import six
-
from scrapy.exceptions import NotConfigured
def is_botocore():
try:
- import botocore
+ import botocore # noqa: F401
return True
except ImportError:
- if six.PY2:
- try:
- import boto
- return False
- except ImportError:
- raise NotConfigured('missing botocore or boto library')
- else:
- raise NotConfigured('missing botocore library')
+ raise NotConfigured('missing botocore library')
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 435e9a6b3..23306ca28 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,11 +1,9 @@
import os
import sys
import numbers
+from configparser import ConfigParser
from operator import itemgetter
-import six
-from six.moves.configparser import SafeConfigParser
-
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
- for k, v in six.iteritems(compdict):
+ for k, v in compdict.items():
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
@@ -33,27 +31,27 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
return compbs
else:
_check_components(compdict)
- return {convert(k): v for k, v in six.iteritems(compdict)}
+ return {convert(k): v for k, v in compdict.items()}
def _validate_values(compdict):
"""Fail if a value in the components dict is not a real number or None."""
- for name, value in six.iteritems(compdict):
+ for name, value in compdict.items():
if value is not None and not isinstance(value, numbers.Real):
- raise ValueError('Invalid value {} for component {}, please provide ' \
+ raise ValueError('Invalid value {} for component {}, please provide '
'a real number or None instead'.format(value, name))
- # BEGIN Backwards compatibility for old (base, custom) call signature
+ # BEGIN Backward compatibility for old (base, custom) call signature
if isinstance(custom, (list, tuple)):
_check_components(custom)
return type(custom)(convert(c) for c in custom)
if custom is not None:
compdict.update(custom)
- # END Backwards compatibility
+ # END Backward compatibility
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
- return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+ return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
def arglist_to_dict(arglist):
@@ -92,9 +90,9 @@ def init_env(project='default', set_syspath=True):
def get_config(use_closest=True):
- """Get Scrapy config file as a SafeConfigParser"""
+ """Get Scrapy config file as a ConfigParser"""
sources = get_sources(use_closest)
- cfg = SafeConfigParser()
+ cfg = ConfigParser()
cfg.read(sources)
return cfg
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 2e9981556..7eb40f0ce 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,6 +1,7 @@
from functools import wraps
from collections import OrderedDict
+
def _embed_ipython_shell(namespace={}, banner=''):
"""Start an IPython Shell"""
try:
@@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''):
shell()
return wrapper
+
def _embed_bpython_shell(namespace={}, banner=''):
"""Start a bpython shell"""
import bpython
@@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''):
bpython.embed(locals_=namespace, banner=banner)
return wrapper
+
def _embed_ptpython_shell(namespace={}, banner=''):
"""Start a ptpython shell"""
import ptpython.repl
@@ -40,21 +43,23 @@ def _embed_ptpython_shell(namespace={}, banner=''):
ptpython.repl.embed(locals=namespace)
return wrapper
+
def _embed_standard_shell(namespace={}, banner=''):
"""Start a standard python shell"""
import code
- try: # readline module is only available on unix systems
+ try: # readline module is only available on unix systems
import readline
except ImportError:
pass
else:
- import rlcompleter
+ import rlcompleter # noqa: F401
readline.parse_and_bind("tab:complete")
@wraps(_embed_standard_shell)
def wrapper(namespace=namespace, banner=''):
code.interact(banner=banner, local=namespace)
return wrapper
+
DEFAULT_PYTHON_SHELLS = OrderedDict([
('ptpython', _embed_ptpython_shell),
('ipython', _embed_ipython_shell),
@@ -62,13 +67,14 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([
('python', _embed_standard_shell),
])
+
def get_shell_embed_func(shells=None, known_shells=None):
"""Return the first acceptable shell-embed function
from a given list of shell names.
"""
- if shells is None: # list, preference order of shells
+ if shells is None: # list, preference order of shells
shells = DEFAULT_PYTHON_SHELLS.keys()
- if known_shells is None: # available embeddable shells
+ if known_shells is None: # available embeddable shells
known_shells = DEFAULT_PYTHON_SHELLS.copy()
for shell in shells:
if shell in known_shells:
@@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None):
except ImportError:
continue
+
def start_python_console(namespace=None, banner='', shells=None):
"""Start Python console bound to the given namespace.
Readline support and tab completion will be used on Unix, if available.
@@ -90,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None):
shell = get_shell_embed_func(shells)
if shell is not None:
shell(namespace=namespace, banner=banner)
- except SystemExit: # raised when using exit() in python code.interact
+ except SystemExit: # raised when using exit() in python code.interact
pass
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
new file mode 100644
index 000000000..16639356e
--- /dev/null
+++ b/scrapy/utils/curl.py
@@ -0,0 +1,94 @@
+import argparse
+import warnings
+from shlex import split
+from http.cookies import SimpleCookie
+from urllib.parse import urlparse
+
+from w3lib.http import basic_auth_header
+
+
+class CurlParser(argparse.ArgumentParser):
+ def error(self, message):
+ error_msg = \
+ 'There was an error parsing the curl command: {}'.format(message)
+ raise ValueError(error_msg)
+
+
+curl_parser = CurlParser()
+curl_parser.add_argument('url')
+curl_parser.add_argument('-H', '--header', dest='headers', action='append')
+curl_parser.add_argument('-X', '--request', dest='method', default='get')
+curl_parser.add_argument('-d', '--data', dest='data')
+curl_parser.add_argument('-u', '--user', dest='auth')
+
+
+safe_to_ignore_arguments = [
+ ['--compressed'],
+ # `--compressed` argument is not safe to ignore, but it's included here
+ # because the `HttpCompressionMiddleware` is enabled by default
+ ['-s', '--silent'],
+ ['-v', '--verbose'],
+ ['-#', '--progress-bar']
+]
+
+for argument in safe_to_ignore_arguments:
+ curl_parser.add_argument(*argument, action='store_true')
+
+
+def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
+ """Convert a cURL command syntax to Request kwargs.
+
+ :param str curl_command: string containing the curl command
+ :param bool ignore_unknown_options: If true, only a warning is emitted when
+ cURL options are unknown. Otherwise raises an error. (default: True)
+ :return: dictionary of Request kwargs
+ """
+
+ curl_args = split(curl_command)
+
+ if curl_args[0] != 'curl':
+ raise ValueError('A curl command must start with "curl"')
+
+ parsed_args, argv = curl_parser.parse_known_args(curl_args[1:])
+
+ if argv:
+ msg = 'Unrecognized options: {}'.format(', '.join(argv))
+ if ignore_unknown_options:
+ warnings.warn(msg)
+ else:
+ raise ValueError(msg)
+
+ url = parsed_args.url
+
+ # curl automatically prepends 'http' if the scheme is missing, but Request
+ # needs the scheme to work
+ parsed_url = urlparse(url)
+ if not parsed_url.scheme:
+ url = 'http://' + url
+
+ result = {'method': parsed_args.method.upper(), 'url': url}
+
+ headers = []
+ cookies = {}
+ for header in parsed_args.headers or ():
+ name, val = header.split(':', 1)
+ name = name.strip()
+ val = val.strip()
+ if name.title() == 'Cookie':
+ for name, morsel in SimpleCookie(val).items():
+ cookies[name] = morsel.value
+ else:
+ headers.append((name, val))
+
+ if parsed_args.auth:
+ user, password = parsed_args.auth.split(':', 1)
+ headers.append(('Authorization', basic_auth_header(user, password)))
+
+ if headers:
+ result['headers'] = headers
+ if cookies:
+ result['cookies'] = cookies
+ if parsed_args.data:
+ result['body'] = parsed_args.data
+
+ return result
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index eb373c501..a52bbc70e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -5,10 +5,11 @@ Python Standard Library.
This module must not depend on any module outside the Standard Library.
"""
+import collections
import copy
-import six
import warnings
-from collections import OrderedDict, Mapping
+import weakref
+from collections.abc import Mapping
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -149,7 +150,7 @@ class MultiValueDict(dict):
self.setlistdefault(key, []).append(value)
except TypeError:
raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
- for key, value in six.iteritems(kwargs):
+ for key, value in kwargs.items():
self.setlistdefault(key, []).append(value)
@@ -236,64 +237,10 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
-class MergeDict(object):
- """
- A simple class for creating new "virtual" dictionaries that actually look
- up values in more than one dictionary, passed in the constructor.
-
- If a key appears in more than one of the given dictionaries, only the
- first occurrence will be used.
- """
- def __init__(self, *dicts):
- self.dicts = dicts
-
- def __getitem__(self, key):
- for dict_ in self.dicts:
- try:
- return dict_[key]
- except KeyError:
- pass
- raise KeyError
-
- def __copy__(self):
- return self.__class__(*self.dicts)
-
- def get(self, key, default=None):
- try:
- return self[key]
- except KeyError:
- return default
-
- def getlist(self, key):
- for dict_ in self.dicts:
- if key in dict_.keys():
- return dict_.getlist(key)
- return []
-
- def items(self):
- item_list = []
- for dict_ in self.dicts:
- item_list.extend(dict_.items())
- return item_list
-
- def has_key(self, key):
- for dict_ in self.dicts:
- if key in dict_:
- return True
- return False
-
- __contains__ = has_key
-
- def copy(self):
- """Returns a copy of this object."""
- return self.__copy__()
-
-
-class LocalCache(OrderedDict):
+class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.
Older items expires first.
-
"""
def __init__(self, limit=None):
@@ -301,11 +248,41 @@ class LocalCache(OrderedDict):
self.limit = limit
def __setitem__(self, key, value):
- while len(self) >= self.limit:
- self.popitem(last=False)
+ if self.limit:
+ while len(self) >= self.limit:
+ self.popitem(last=False)
super(LocalCache, self).__setitem__(key, value)
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
+ """
+ A weakref.WeakKeyDictionary implementation that uses LocalCache as its
+ underlying data structure, making it ordered and capable of being size-limited.
+
+ Useful for memoization, while avoiding keeping received
+ arguments in memory only because of the cached references.
+
+ Note: like LocalCache and unlike weakref.WeakKeyDictionary,
+ it cannot be instantiated with an initial dictionary.
+ """
+
+ def __init__(self, limit=None):
+ super(LocalWeakReferencedCache, self).__init__()
+ self.data = LocalCache(limit=limit)
+
+ def __setitem__(self, key, value):
+ try:
+ super(LocalWeakReferencedCache, self).__setitem__(key, value)
+ except TypeError:
+ pass # key is not weak-referenceable, skip caching
+
+ def __getitem__(self, key):
+ try:
+ return super(LocalWeakReferencedCache, self).__getitem__(key)
+ except TypeError:
+ return None # key is not weak-referenceable, it's not cached
+
+
class SequenceExclude(object):
"""Object to test if an item is NOT within some sequence."""
diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py
deleted file mode 100644
index e8c8eae39..000000000
--- a/scrapy/utils/decorator.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.utils.decorator` is deprecated, "
- "use `scrapy.utils.decorators` instead",
- ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.utils.decorators import *
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 38bee1a6c..2e2c7adc1 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -34,6 +34,7 @@ def defers(func):
return defer.maybeDeferred(func, *a, **kw)
return wrapped
+
def inthread(func):
"""Decorator to call a function in a thread and return a deferred with the
result
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bcf209511..62b43a96c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -1,11 +1,16 @@
"""
Helper functions for dealing with Twisted deferreds
"""
+import asyncio
+from functools import wraps
+import inspect
-from twisted.internet import defer, reactor, task
+from twisted.internet import defer, task
from twisted.python import failure
from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
+
def defer_fail(_failure):
"""Same as twisted.internet.defer.fail but delay calling errback until
@@ -14,10 +19,12 @@ def defer_fail(_failure):
It delays by 100ms so reactor has a chance to go through readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
+ from twisted.internet import reactor
d = defer.Deferred()
reactor.callLater(0.1, d.errback, _failure)
return d
+
def defer_succeed(result):
"""Same as twisted.internet.defer.succeed but delay calling callback until
next reactor loop
@@ -25,10 +32,12 @@ def defer_succeed(result):
It delays by 100ms so reactor has a chance to go trough readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
+ from twisted.internet import reactor
d = defer.Deferred()
reactor.callLater(0.1, d.callback, result)
return d
+
def defer_result(result):
if isinstance(result, defer.Deferred):
return result
@@ -37,6 +46,7 @@ def defer_result(result):
else:
return defer_succeed(result)
+
def mustbe_deferred(f, *args, **kw):
"""Same as twisted.internet.defer.maybeDeferred, but delay calling
callback/errback to next reactor loop
@@ -48,11 +58,12 @@ def mustbe_deferred(f, *args, **kw):
# exception in Scrapy - see #125
except IgnoreRequest as e:
return defer_fail(failure.Failure(e))
- except:
+ except Exception:
return defer_fail(failure.Failure())
else:
return defer_result(result)
+
def parallel(iterable, count, callable, *args, **named):
"""Execute a callable over the objects in the given iterable, in parallel,
using no more than ``count`` concurrent calls.
@@ -63,6 +74,7 @@ def parallel(iterable, count, callable, *args, **named):
work = (callable(elem, *args, **named) for elem in iterable)
return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
+
def process_chain(callbacks, input, *a, **kw):
"""Return a Deferred built by chaining the given callbacks"""
d = defer.Deferred()
@@ -71,6 +83,7 @@ def process_chain(callbacks, input, *a, **kw):
d.callback(input)
return d
+
def process_chain_both(callbacks, errbacks, input, *a, **kw):
"""Return a Deferred built by chaining the given callbacks and errbacks"""
d = defer.Deferred()
@@ -83,6 +96,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw):
d.callback(input)
return d
+
def process_parallel(callbacks, input, *a, **kw):
"""Return a Deferred with the output of all successful calls to the given
callbacks
@@ -92,6 +106,7 @@ def process_parallel(callbacks, input, *a, **kw):
d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
return d
+
def iter_errback(iterable, errback, *a, **kw):
"""Wraps an iterable calling an errback if an error is caught while
iterating it.
@@ -102,5 +117,39 @@ def iter_errback(iterable, errback, *a, **kw):
yield next(it)
except StopIteration:
break
- except:
+ except Exception:
errback(failure.Failure(), *a, **kw)
+
+
+def _isfuture(o):
+ # workaround for Python before 3.5.3 not having asyncio.isfuture
+ if hasattr(asyncio, 'isfuture'):
+ return asyncio.isfuture(o)
+ return isinstance(o, asyncio.Future)
+
+
+def deferred_from_coro(o):
+ """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
+ if isinstance(o, defer.Deferred):
+ return o
+ if _isfuture(o) or inspect.isawaitable(o):
+ if not is_asyncio_reactor_installed():
+ # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
+ # that use asyncio, e.g. "await asyncio.sleep(1)"
+ return defer.ensureDeferred(o)
+ else:
+ # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
+ return defer.Deferred.fromFuture(asyncio.ensure_future(o))
+ return o
+
+
+def deferred_f_from_coro_f(coro_f):
+ """ Converts a coroutine function into a function that returns a Deferred.
+
+ The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
+ This is useful for callback chains, as callback functions are called with the previous callback result.
+ """
+ @wraps(coro_f)
+ def f(*coro_args, **coro_kwargs):
+ return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
+ return f
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index f76161a68..2d3db431d 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -124,26 +124,7 @@ def _clspath(cls, forced=None):
DEPRECATION_RULES = [
- ('scrapy.contrib_exp.downloadermiddleware.decompression.', 'scrapy.downloadermiddlewares.decompression.'),
- ('scrapy.contrib_exp.iterators.', 'scrapy.utils.iterators.'),
- ('scrapy.contrib.downloadermiddleware.', 'scrapy.downloadermiddlewares.'),
- ('scrapy.contrib.exporter.', 'scrapy.exporters.'),
- ('scrapy.contrib.linkextractors.', 'scrapy.linkextractors.'),
- ('scrapy.contrib.loader.processor.', 'scrapy.loader.processors.'),
- ('scrapy.contrib.loader.', 'scrapy.loader.'),
- ('scrapy.contrib.pipeline.', 'scrapy.pipelines.'),
- ('scrapy.contrib.spidermiddleware.', 'scrapy.spidermiddlewares.'),
- ('scrapy.contrib.spiders.', 'scrapy.spiders.'),
- ('scrapy.contrib.', 'scrapy.extensions.'),
- ('scrapy.command.', 'scrapy.commands.'),
- ('scrapy.dupefilter.', 'scrapy.dupefilters.'),
- ('scrapy.linkextractor.', 'scrapy.linkextractors.'),
('scrapy.telnet.', 'scrapy.extensions.telnet.'),
- ('scrapy.spider.', 'scrapy.spiders.'),
- ('scrapy.squeue.', 'scrapy.squeues.'),
- ('scrapy.statscol.', 'scrapy.statscollectors.'),
- ('scrapy.utils.decorator.', 'scrapy.utils.decorators.'),
- ('scrapy.spidermanager.SpiderManager', 'scrapy.spiderloader.SpiderLoader'),
]
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index f6a6c4645..9735220ef 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -2,10 +2,10 @@
pprint and pformat wrappers with colorization support
"""
-from __future__ import print_function
import sys
from pprint import pformat as pformat_
+
def _colorize(text, colorize=True):
if not colorize or not sys.stdout.isatty():
return text
@@ -17,8 +17,10 @@ def _colorize(text, colorize=True):
except ImportError:
return text
+
def pformat(obj, *args, **kwargs):
return _colorize(pformat_(obj), kwargs.pop('colorize', True))
+
def pprint(obj, *args, **kwargs):
print(pformat(obj, *args, **kwargs))
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 11dd36d91..267c7ecd1 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -1,7 +1,8 @@
"""Some debugging functions for working with the Scrapy engine"""
-from __future__ import print_function
-from time import time # used in global tests code
+# used in global tests code
+from time import time # noqa: F401
+
def get_engine_status(engine):
"""Return a report of the current engine status"""
@@ -32,6 +33,7 @@ def get_engine_status(engine):
return checks
+
def format_engine_status(engine=None):
checks = get_engine_status(engine)
s = "Execution engine status\n\n"
@@ -41,5 +43,6 @@ def format_engine_status(engine=None):
return s
+
def print_engine_status(engine):
print(format_engine_status(engine))
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index f255d436f..f07bdd748 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -1,8 +1,11 @@
-from ftplib import error_perm
+import posixpath
+
+from ftplib import error_perm, FTP
from posixpath import dirname
+
def ftp_makedirs_cwd(ftp, path, first_call=True):
- """Set the current directory of the FTP connection given in the `ftp`
+ """Set the current directory of the FTP connection given in the ``ftp``
argument (as a ftplib.FTP object), creating all parent directories if they
don't exist. The ftplib.FTP object must be already connected and logged in.
"""
@@ -13,3 +16,20 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
ftp.mkd(path)
if first_call:
ftp.cwd(path)
+
+
+def ftp_store_file(
+ *, path, file, host, port,
+ username, password, use_active_mode=False):
+ """Opens a FTP connection with passed credentials,sets current directory
+ to the directory extracted from given path, then uploads the file to server
+ """
+ with FTP() as ftp:
+ ftp.connect(host, port)
+ ftp.login(username, password)
+ if use_active_mode:
+ ftp.set_pasv(False)
+ file.seek(0)
+ dirname, filename = posixpath.split(path)
+ ftp_makedirs_cwd(ftp, dirname)
+ ftp.storbinary('STOR %s' % filename, file)
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 16c9ce539..9672e28da 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,27 +1,19 @@
+from gzip import GzipFile
+from io import BytesIO
+import re
import struct
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
-from gzip import GzipFile
+from scrapy.utils.decorators import deprecated
-import six
-import re
# - Python>=3.5 GzipFile's read() has issues returning leftover
# uncompressed data when input is corrupted
# (regression or bug-fix compared to Python 3.4)
# - read1(), which fetches data before raising EOFError on next call
# works here but is only available from Python>=3.3
-# - scrapy does not support Python 3.2
-# - Python 2.7 GzipFile works fine with standard read() + extrabuf
-if six.PY2:
- def read1(gzf, size=-1):
- return gzf.read(size)
-else:
- def read1(gzf, size=-1):
- return gzf.read1(size)
+@deprecated('GzipFile.read1')
+def read1(gzf, size=-1):
+ return gzf.read1(size)
def gunzip(data):
@@ -30,29 +22,31 @@ def gunzip(data):
This is resilient to CRC checksum errors.
"""
f = GzipFile(fileobj=BytesIO(data))
- output = b''
+ output_list = []
chunk = b'.'
while chunk:
try:
- chunk = read1(f, 8196)
- output += chunk
+ chunk = f.read1(8196)
+ output_list.append(chunk)
except (IOError, EOFError, struct.error):
# complete only if there is some data, otherwise re-raise
# see issue 87 about catching struct.error
- # some pages are quite small so output is '' and f.extrabuf
+ # some pages are quite small so output_list is empty and f.extrabuf
# contains the whole page content
- if output or getattr(f, 'extrabuf', None):
+ if output_list or getattr(f, 'extrabuf', None):
try:
- output += f.extrabuf[-f.extrasize:]
+ output_list.append(f.extrabuf[-f.extrasize:])
finally:
break
else:
raise
- return output
+ return b''.join(output_list)
_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
+
+@deprecated
def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""
ctype = response.headers.get('Content-Type', b'')
diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py
index 7cc8d1884..bab262393 100644
--- a/scrapy/utils/http.py
+++ b/scrapy/utils/http.py
@@ -4,8 +4,19 @@ Transitional module for moving to the w3lib library.
For new code, always import from w3lib.http instead of this module
"""
-from w3lib.http import *
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import deprecated
+from w3lib.http import * # noqa: F401
+
+
+warnings.warn("Module `scrapy.utils.http` is deprecated, "
+ "Please import from `w3lib.http` instead.",
+ ScrapyDeprecationWarning, stacklevel=2)
+
+
+@deprecated
def decode_chunked_transfer(chunked_body):
"""Parsed body received with chunked transfer encoding, and return the
decoded body.
@@ -23,4 +34,3 @@ def decode_chunked_transfer(chunked_body):
body += t[:size]
t = t[size+2:]
return body
-
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index b4c929b0e..c8d4391b1 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,10 +1,12 @@
"""Helper functions for scrapy.http objects (Request, Response)"""
import weakref
+from urllib.parse import urlparse
-from six.moves.urllib.parse import urlparse
_urlparse_cache = weakref.WeakKeyDictionary()
+
+
def urlparse_cached(request_or_response):
"""Return urlparse.urlparse caching the result, where the argument can be a
Request or Response object
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 73857b410..3c0cb68c3 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,17 +1,13 @@
-import re
import csv
import logging
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
+import re
from io import StringIO
-import six
from scrapy.http import TextResponse, Response
from scrapy.selector import Selector
from scrapy.utils.python import re_rsearch, to_unicode
+
logger = logging.getLogger(__name__)
@@ -64,7 +60,7 @@ class _StreamReader(object):
self._text, self.encoding = obj.body, obj.encoding
else:
self._text, self.encoding = obj, 'utf-8'
- self._is_unicode = isinstance(self._text, six.text_type)
+ self._is_unicode = isinstance(self._text, str)
def read(self, n=65535):
self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -98,14 +94,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
"""
encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or 'utf-8'
- def _getrow(csv_r):
- return [to_unicode(field, encoding) for field in next(csv_r)]
- # Python 3 csv reader input object needs to return strings
- if six.PY3:
- lines = StringIO(_body_or_str(obj, unicode=True))
- else:
- lines = BytesIO(_body_or_str(obj, unicode=False))
+ def row_to_unicode(row_):
+ return [to_unicode(field, encoding) for field in row_]
+
+ lines = StringIO(_body_or_str(obj, unicode=True))
kwargs = {}
if delimiter: kwargs["delimiter"] = delimiter
@@ -113,10 +106,14 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
csv_r = csv.reader(lines, **kwargs)
if not headers:
- headers = _getrow(csv_r)
+ try:
+ row = next(csv_r)
+ except StopIteration:
+ return
+ headers = row_to_unicode(row)
- while True:
- row = _getrow(csv_r)
+ for row in csv_r:
+ row = row_to_unicode(row)
if len(row) != len(headers):
logger.warning("ignoring row %(csvlnum)d (length: %(csvrow)d, "
"should be: %(csvheader)d)",
@@ -128,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
- expected_types = (Response, six.text_type, six.binary_type)
+ expected_types = (Response, str, bytes)
assert isinstance(obj, expected_types), \
"obj must be %s, not %s" % (
" or ".join(t.__name__ for t in expected_types),
@@ -140,7 +137,7 @@ def _body_or_str(obj, unicode=True):
return obj.text
else:
return obj.body.decode('utf-8')
- elif isinstance(obj, six.text_type):
+ elif isinstance(obj, str):
return obj if unicode else obj.encode('utf-8')
else:
return obj.decode('utf-8') if unicode else obj
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index 389fde73a..4f1e601fc 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -1,5 +1,6 @@
import os
+
def job_dir(settings):
path = settings['JOBDIR']
if path and not os.path.exists(path):
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 828880709..e4cf0196b 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -11,6 +11,7 @@ from twisted.python import log as twisted_log
import scrapy
from scrapy.settings import Settings
from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncio import is_asyncio_reactor_installed
from scrapy.utils.versions import scrapy_components_versions
@@ -32,7 +33,7 @@ class TopLevelFormatter(logging.Filter):
Since it can't be set for just one logger (it won't propagate for its
children), it's going to be set in the root handler, with a parametrized
- `loggers` list where it should act.
+ ``loggers`` list where it should act.
"""
def __init__(self, loggers=None):
@@ -148,6 +149,8 @@ def log_scrapy_info(settings):
{'versions': ", ".join("%s %s" % (name, version)
for name, version in scrapy_components_versions()
if name != "Scrapy")})
+ if is_asyncio_reactor_installed():
+ logger.debug("Asyncio reactor is installed")
class StreamLogger(object):
diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
index 977133f4e..9728c542a 100644
--- a/scrapy/utils/markup.py
+++ b/scrapy/utils/markup.py
@@ -3,5 +3,12 @@ Transitional module for moving to the w3lib library.
For new code, always import from w3lib.html instead of this module
"""
+import warnings
-from w3lib.html import *
+from scrapy.exceptions import ScrapyDeprecationWarning
+from w3lib.html import * # noqa: F401
+
+
+warnings.warn("Module `scrapy.utils.markup` is deprecated. "
+ "Please import from `w3lib.html` instead.",
+ ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 35f855007..cb0ee5af3 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,17 +1,23 @@
"""Helper functions which don't fit anywhere else"""
+import ast
+import inspect
+import os
import re
import hashlib
+import warnings
+from contextlib import contextmanager
from importlib import import_module
from pkgutil import iter_modules
+from textwrap import dedent
-import six
from w3lib.html import replace_entities
+from scrapy.utils.datatypes import LocalWeakReferencedCache
from scrapy.utils.python import flatten, to_unicode
from scrapy.item import BaseItem
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
def arg_to_iter(arg):
@@ -81,16 +87,16 @@ def extract_regex(regex, text, encoding='utf-8'):
* if the regex doesn't contain any group the entire regex matching is returned
"""
- if isinstance(regex, six.string_types):
+ if isinstance(regex, str):
regex = re.compile(regex, re.UNICODE)
try:
strings = [regex.search(text).group('extract')] # named group
- except:
+ except Exception:
strings = regex.findall(text) # full regex or numbered groups
strings = flatten(strings)
- if isinstance(text, six.text_type):
+ if isinstance(text, str):
return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
else:
return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
@@ -116,4 +122,88 @@ def md5sum(file):
def rel_has_nofollow(rel):
"""Return True if link rel attribute has nofollow type"""
- return True if rel is not None and 'nofollow' in rel.split() else False
+ return rel is not None and 'nofollow' in rel.split()
+
+
+def create_instance(objcls, settings, crawler, *args, **kwargs):
+ """Construct a class instance using its ``from_crawler`` or
+ ``from_settings`` constructors, if available.
+
+ At least one of ``settings`` and ``crawler`` needs to be different from
+ ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
+ If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
+ tried.
+
+ ``*args`` and ``**kwargs`` are forwarded to the constructors.
+
+ Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
+ """
+ if settings is None:
+ if crawler is None:
+ raise ValueError("Specify at least one of settings and crawler.")
+ settings = crawler.settings
+ if crawler and hasattr(objcls, 'from_crawler'):
+ return objcls.from_crawler(crawler, *args, **kwargs)
+ elif hasattr(objcls, 'from_settings'):
+ return objcls.from_settings(settings, *args, **kwargs)
+ else:
+ return objcls(*args, **kwargs)
+
+
+@contextmanager
+def set_environ(**kwargs):
+ """Temporarily set environment variables inside the context manager and
+ fully restore previous environment afterwards
+ """
+
+ original_env = {k: os.environ.get(k) for k in kwargs}
+ os.environ.update(kwargs)
+ try:
+ yield
+ finally:
+ for k, v in original_env.items():
+ if v is None:
+ del os.environ[k]
+ else:
+ os.environ[k] = v
+
+
+_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
+
+
+def is_generator_with_return_value(callable):
+ """
+ Returns True if a callable is a generator function which includes a
+ 'return' statement with a value different than None, False otherwise
+ """
+ if callable in _generator_callbacks_cache:
+ return _generator_callbacks_cache[callable]
+
+ def returns_none(return_node):
+ value = return_node.value
+ return value is None or isinstance(value, ast.NameConstant) and value.value is None
+
+ if inspect.isgeneratorfunction(callable):
+ tree = ast.parse(dedent(inspect.getsource(callable)))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Return) and not returns_none(node):
+ _generator_callbacks_cache[callable] = True
+ return _generator_callbacks_cache[callable]
+
+ _generator_callbacks_cache[callable] = False
+ return _generator_callbacks_cache[callable]
+
+
+def warn_on_generator_with_return_value(spider, callable):
+ """
+ Logs a warning if a callable is a generator function and includes
+ a 'return' statement with a value different than None
+ """
+ if is_generator_with_return_value(callable):
+ warnings.warn(
+ 'The "{}.{}" method is a generator and includes a "return" statement with a '
+ 'value different than None. This could lead to unexpected behaviour. Please see '
+ 'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
+ 'for details about the semantics of the "return" statement within generators'
+ .format(spider.__class__.__name__, callable.__name__), stacklevel=2,
+ )
diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py
index ec26c0866..5dcf791b8 100644
--- a/scrapy/utils/multipart.py
+++ b/scrapy/utils/multipart.py
@@ -3,5 +3,13 @@ Transitional module for moving to the w3lib library.
For new code, always import from w3lib.form instead of this module
"""
+import warnings
-from w3lib.form import *
+from scrapy.exceptions import ScrapyDeprecationWarning
+from w3lib.form import * # noqa: F401
+
+
+warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
+ "If you're using `encode_multipart` function, please use "
+ "`urllib3.filepost.encode_multipart_formdata` instead",
+ ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index f87d5a803..45c9cef0c 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,9 +1,5 @@
-
-from __future__ import absolute_import
import signal
-from twisted.internet import reactor
-
signal_names = {}
for signame in dir(signal):
@@ -19,6 +15,7 @@ def install_shutdown_handlers(function, override_sigint=True):
SIGINT handler won't be install if there is already a handler in place
(e.g. Pdb)
"""
+ from twisted.internet import reactor
reactor._handleSignals()
signal.signal(signal.SIGTERM, function)
if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 95c6a8035..f28c2eaa1 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,5 +1,5 @@
import os
-from six.moves import cPickle as pickle
+import pickle
import warnings
from importlib import import_module
@@ -7,7 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+
ENVVAR = 'SCRAPY_SETTINGS_MODULE'
DATADIR_CFG_SECTION = 'datadir'
@@ -70,6 +71,9 @@ def get_project_settings():
# XXX: remove this hack
pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
if pickled_settings:
+ warnings.warn("Use of environment variable "
+ "'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' "
+ "is deprecated.", ScrapyDeprecationWarning)
settings.setdict(pickle.loads(pickled_settings), priority='project')
# XXX: deprecate and remove this functionality
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 732ca13a0..8d829c5a5 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,8 +7,8 @@ import re
import inspect
import weakref
import errno
-import six
from functools import partial, wraps
+from itertools import chain
import sys
from scrapy.utils.decorators import deprecated
@@ -64,10 +64,10 @@ def is_listlike(x):
True
>>> is_listlike((x for x in range(3)))
True
- >>> is_listlike(six.moves.xrange(5))
+ >>> is_listlike(range(5))
True
"""
- return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+ return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
def unique(list_, key=lambda x: x):
@@ -83,26 +83,13 @@ def unique(list_, key=lambda x: x):
return result
-@deprecated("scrapy.utils.python.to_unicode")
-def str_to_unicode(text, encoding=None, errors='strict'):
- """ This function is deprecated.
- Please use scrapy.utils.python.to_unicode. """
- return to_unicode(text, encoding, errors)
-
-
-@deprecated("scrapy.utils.python.to_bytes")
-def unicode_to_str(text, encoding=None, errors='strict'):
- """ This function is deprecated. Please use scrapy.utils.python.to_bytes """
- return to_bytes(text, encoding, errors)
-
-
def to_unicode(text, encoding=None, errors='strict'):
- """Return the unicode representation of a bytes object `text`. If `text`
- is already an unicode object, return it as-is."""
- if isinstance(text, six.text_type):
+ """Return the unicode representation of a bytes object ``text``. If
+ ``text`` is already an unicode object, return it as-is."""
+ if isinstance(text, str):
return text
- if not isinstance(text, (bytes, six.text_type)):
- raise TypeError('to_unicode must receive a bytes, str or unicode '
+ if not isinstance(text, (bytes, str)):
+ raise TypeError('to_unicode must receive a bytes or str '
'object, got %s' % type(text).__name__)
if encoding is None:
encoding = 'utf-8'
@@ -110,25 +97,22 @@ def to_unicode(text, encoding=None, errors='strict'):
def to_bytes(text, encoding=None, errors='strict'):
- """Return the binary representation of `text`. If `text`
+ """Return the binary representation of ``text``. If ``text``
is already a bytes object, return it as-is."""
if isinstance(text, bytes):
return text
- if not isinstance(text, six.string_types):
- raise TypeError('to_bytes must receive a unicode, str or bytes '
+ if not isinstance(text, str):
+ raise TypeError('to_bytes must receive a str or bytes '
'object, got %s' % type(text).__name__)
if encoding is None:
encoding = 'utf-8'
return text.encode(encoding, errors)
+@deprecated('to_unicode')
def to_native_str(text, encoding=None, errors='strict'):
- """ Return str representation of `text`
- (bytes in Python 2.x and unicode in Python 3.x). """
- if six.PY2:
- return to_bytes(text, encoding, errors)
- else:
- return to_unicode(text, encoding, errors)
+ """ Return str representation of ``text``. """
+ return to_unicode(text, encoding, errors)
def re_rsearch(pattern, text, chunk_size=1024):
@@ -153,7 +137,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
yield (text[offset:], offset)
yield (text, 0)
- if isinstance(pattern, six.string_types):
+ if isinstance(pattern, str):
pattern = re.compile(pattern)
for chunk, offset in _chunk_iter():
@@ -177,9 +161,10 @@ def memoizemethod_noargs(method):
return new_method
-_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
+
@deprecated("scrapy.utils.python.binary_is_text")
def isbinarytext(text):
""" This function is deprecated.
@@ -189,7 +174,7 @@ def isbinarytext(text):
def binary_is_text(data):
- """ Returns `True` if the given ``data`` argument (a ``bytes`` object)
+ """ Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
does not contain unprintable control characters.
"""
if not isinstance(data, bytes):
@@ -201,7 +186,7 @@ def _getargspec_py23(func):
"""_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
defaults)
- Identical to inspect.getargspec() in python2, but uses
+ Was identical to inspect.getargspec() in python2, but uses
inspect.getfullargspec() for python3 behind the scenes to avoid
DeprecationWarning.
@@ -211,9 +196,6 @@ def _getargspec_py23(func):
>>> _getargspec_py23(f)
ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
"""
- if six.PY2:
- return inspect.getargspec(func)
-
return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
@@ -314,14 +296,14 @@ class WeakKeyCache(object):
@deprecated
def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
"""Return a (new) dict with unicode keys (and values when "keys_only" is
- False) of the given dict converted to strings. `dct_or_tuples` can be a
- dict or a list of tuples, like any dict constructor supports.
+ False) of the given dict converted to strings. ``dct_or_tuples`` can be a
+ dict or a list of tuples, like any dict ``__init__`` method supports.
"""
d = {}
- for k, v in six.iteritems(dict(dct_or_tuples)):
- k = k.encode(encoding) if isinstance(k, six.text_type) else k
+ for k, v in dict(dct_or_tuples).items():
+ k = k.encode(encoding) if isinstance(k, str) else k
if not keys_only:
- v = v.encode(encoding) if isinstance(v, six.text_type) else v
+ v = v.encode(encoding) if isinstance(v, str) else v
d[k] = v
return d
@@ -357,13 +339,13 @@ def retry_on_eintr(function, *args, **kw):
def without_none_values(iterable):
- """Return a copy of `iterable` with all `None` entries removed.
+ """Return a copy of ``iterable`` with all ``None`` entries removed.
- If `iterable` is a mapping, return a dictionary where all pairs that have
- value `None` have been removed.
+ If ``iterable`` is a mapping, return a dictionary where all pairs that have
+ value ``None`` have been removed.
"""
try:
- return {k: v for k, v in six.iteritems(iterable) if v is not None}
+ return {k: v for k, v in iterable.items() if v is not None}
except AttributeError:
return type(iterable)((v for v in iterable if v is not None))
@@ -387,3 +369,24 @@ if hasattr(sys, "pypy_version_info"):
else:
def garbage_collect():
gc.collect()
+
+
+class MutableChain(object):
+ """
+ Thin wrapper around itertools.chain, allowing to add iterables "in-place"
+ """
+ def __init__(self, *args):
+ self.data = chain(*args)
+
+ def extend(self, *iterables):
+ self.data = chain(self.data, *iterables)
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self.data)
+
+ @deprecated("scrapy.utils.python.MutableChain.__next__")
+ def next(self):
+ return self.__next__()
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 83186a372..b98fff6ec 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,12 +1,14 @@
-from twisted.internet import reactor, error
+from twisted.internet import error
+
def listen_tcp(portrange, host, factory):
"""Like reactor.listenTCP but tries different ports in a range."""
+ from twisted.internet import reactor
assert len(portrange) <= 2, "invalid portrange: %s" % portrange
- if not hasattr(portrange, '__iter__'):
- return reactor.listenTCP(portrange, factory, interface=host)
if not portrange:
return reactor.listenTCP(0, factory, interface=host)
+ if not hasattr(portrange, '__iter__'):
+ return reactor.listenTCP(portrange, factory, interface=host)
if len(portrange) == 1:
return reactor.listenTCP(portrange[0], factory, interface=host)
for x in range(portrange[0], portrange[1]+1):
@@ -29,6 +31,7 @@ class CallLaterOnce(object):
self._call = None
def schedule(self, delay=0):
+ from twisted.internet import reactor
if self._call is None:
self._call = reactor.callLater(delay, self)
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 959dddbd5..749bbc387 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,10 +1,8 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
-import six
-
from scrapy.http import Request
-from scrapy.utils.python import to_unicode, to_native_str
+from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
@@ -32,7 +30,8 @@ def request_to_dict(request, spider=None):
'_encoding': request._encoding,
'priority': request.priority,
'dont_filter': request.dont_filter,
- 'flags': request.flags
+ 'flags': request.flags,
+ 'cb_kwargs': request.cb_kwargs,
}
if type(request) is not Request:
d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -53,7 +52,7 @@ def request_from_dict(d, spider=None):
eb = _get_method(spider, eb)
request_cls = load_object(d['_class']) if '_class' in d else Request
return request_cls(
- url=to_native_str(d['url']),
+ url=to_unicode(d['url']),
callback=cb,
errback=eb,
method=d['method'],
@@ -64,18 +63,37 @@ def request_from_dict(d, spider=None):
encoding=d['_encoding'],
priority=d['priority'],
dont_filter=d['dont_filter'],
- flags=d.get('flags'))
+ flags=d.get('flags'),
+ cb_kwargs=d.get('cb_kwargs'),
+ )
+
+
+def _is_private_method(name):
+ return name.startswith('__') and not name.endswith('__')
+
+
+def _mangle_private_name(obj, func, name):
+ qualname = getattr(func, '__qualname__', None)
+ if qualname is None:
+ classname = obj.__class__.__name__.lstrip('_')
+ return '_%s%s' % (classname, name)
+ else:
+ splits = qualname.split('.')
+ return '_%s%s' % (splits[-2], splits[-1])
def _find_method(obj, func):
if obj:
try:
- func_self = six.get_method_self(func)
+ func_self = func.__self__
except AttributeError: # func has no __self__
pass
else:
if func_self is obj:
- return six.get_method_function(func).__name__
+ name = func.__func__.__name__
+ if _is_private_method(name):
+ return _mangle_private_name(obj, func, name)
+ return name
raise ValueError("Function %s is not a method of: %s" % (func, obj))
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 50bc3cb1e..356753ab5 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -3,20 +3,21 @@ This module provides some useful functions for working with
scrapy.http.Request objects
"""
-from __future__ import print_function
import hashlib
import weakref
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import urlunparse
from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_native_str
-
from w3lib.url import canonicalize_url
+
from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes, to_unicode
_fingerprint_cache = weakref.WeakKeyDictionary()
-def request_fingerprint(request, include_headers=None):
+
+
+def request_fingerprint(request, include_headers=None, keep_fragments=False):
"""
Return the request fingerprint.
@@ -30,7 +31,7 @@ def request_fingerprint(request, include_headers=None):
and are equivalent (ie. they should return the same response).
Another example are cookies used to store session ids. Suppose the
- following page is only accesible to authenticated users:
+ following page is only accessible to authenticated users:
http://www.example.com/members/offers.html
@@ -42,15 +43,21 @@ def request_fingerprint(request, include_headers=None):
the fingeprint. If you want to include specific headers use the
include_headers argument, which is a list of Request headers to include.
+ Also, servers usually ignore fragments in urls when handling requests,
+ so they are also ignored by default when calculating the fingerprint.
+ If you want to include them, set the keep_fragments argument to True
+ (for instance when handling requests with a headless browser).
+
"""
if include_headers:
include_headers = tuple(to_bytes(h.lower())
for h in sorted(include_headers))
cache = _fingerprint_cache.setdefault(request, {})
- if include_headers not in cache:
+ cache_key = (include_headers, keep_fragments)
+ if cache_key not in cache:
fp = hashlib.sha1()
fp.update(to_bytes(request.method))
- fp.update(to_bytes(canonicalize_url(request.url)))
+ fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
fp.update(request.body or b'')
if include_headers:
for hdr in include_headers:
@@ -58,8 +65,8 @@ def request_fingerprint(request, include_headers=None):
fp.update(hdr)
for v in request.headers.getlist(hdr):
fp.update(v)
- cache[include_headers] = fp.hexdigest()
- return cache[include_headers]
+ cache[cache_key] = fp.hexdigest()
+ return cache[cache_key]
def request_authenticate(request, username, password):
@@ -91,4 +98,4 @@ def referer_str(request):
referrer = request.headers.get('Referer')
if referrer is None:
return referrer
- return to_native_str(referrer, errors='replace')
+ return to_unicode(referrer, errors='replace')
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index bf276b5ca..29fdaaf2c 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -8,19 +8,13 @@ import webbrowser
import tempfile
from twisted.web import http
-from scrapy.utils.python import to_bytes, to_native_str
+from scrapy.utils.python import to_bytes, to_unicode
from w3lib import html
-from scrapy.utils.decorators import deprecated
-
-
-@deprecated
-def body_or_str(*a, **kw):
- from scrapy.utils.iterators import _body_or_str
- return _body_or_str(*a, **kw)
-
_baseurl_cache = weakref.WeakKeyDictionary()
+
+
def get_base_url(response):
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
@@ -31,12 +25,14 @@ def get_base_url(response):
_metaref_cache = weakref.WeakKeyDictionary()
-def get_meta_refresh(response):
+
+
+def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
text = response.text[0:4096]
_metaref_cache[response] = html.get_meta_refresh(text, response.url,
- response.encoding, ignore_tags=('script', 'noscript'))
+ response.encoding, ignore_tags=ignore_tags)
return _metaref_cache[response]
@@ -44,7 +40,7 @@ def response_status_message(status):
"""Return status code plus status text descriptive message
"""
message = http.RESPONSES.get(int(status), "Unknown Status")
- return '%s %s' % (status, to_native_str(message))
+ return '%s %s' % (status, to_unicode(message))
def response_httprepr(response):
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 4742b3e13..2f10cf4de 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
SitemapSpider, its API is subject to change without notice.
"""
+from urllib.parse import urljoin
+
import lxml.etree
-from six.moves.urllib.parse import urljoin
class Sitemap(object):
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 94b24f67e..72775df5c 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,16 +1,16 @@
import logging
import inspect
-import six
-
from scrapy.spiders import Spider
-from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.misc import arg_to_iter
+
logger = logging.getLogger(__name__)
def iterate_spider_output(result):
- return arg_to_iter(result)
+ return arg_to_iter(deferred_from_coro(result))
def iter_spider_classes(module):
@@ -21,13 +21,14 @@ def iter_spider_classes(module):
# singleton in scrapy.spider.spiders
from scrapy.spiders import Spider
- for obj in six.itervalues(vars(module)):
+ for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, Spider) and \
obj.__module__ == module.__name__ and \
getattr(obj, 'name', None):
yield obj
+
def spidercls_for_request(spider_loader, request, default_spidercls=None,
log_none=False, log_multiple=False):
"""Return a spider class that handles the given Request.
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
new file mode 100644
index 000000000..6e81b33ff
--- /dev/null
+++ b/scrapy/utils/ssl.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+
+import OpenSSL
+import OpenSSL._util as pyOpenSSLutil
+
+from scrapy.utils.python import to_unicode
+
+
+# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
+# Using the binding directly, as this code does, requires cryptography 2.4.
+SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
+
+
+def ffi_buf_to_string(buf):
+ return to_unicode(pyOpenSSLutil.ffi.string(buf))
+
+
+def x509name_to_string(x509name):
+ # from OpenSSL.crypto.X509Name.__repr__
+ result_buffer = pyOpenSSLutil.ffi.new("char[]", 512)
+ pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer))
+
+ return ffi_buf_to_string(result_buffer)
+
+
+def get_temp_key_info(ssl_object):
+ if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2
+ return None
+
+ # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
+ temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
+ if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
+ return None
+ temp_key = temp_key_p[0]
+ if temp_key == pyOpenSSLutil.ffi.NULL:
+ return None
+ temp_key = pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)
+ key_info = []
+ key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)
+ if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:
+ key_info.append('RSA')
+ elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH:
+ key_info.append('DH')
+ elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:
+ key_info.append('ECDH')
+ ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)
+ ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
+ nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key))
+ cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
+ if cname == pyOpenSSLutil.ffi.NULL:
+ cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
+ key_info.append(ffi_buf_to_string(cname))
+ else:
+ key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
+ key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key))
+ return ', '.join(key_info)
+
+
+def get_openssl_version():
+ system_openssl = OpenSSL.SSL.SSLeay_version(
+ OpenSSL.SSL.SSLEAY_VERSION
+ ).decode('ascii', errors='replace')
+ return '{} ({})'.format(OpenSSL.version.__version__, system_openssl)
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 1d7bd006c..96ff4b09b 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -18,7 +18,9 @@ def render_templatefile(path, **kwargs):
os.remove(path)
-CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]')
+CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
+
+
def string_camelcase(string):
""" Convert a word to its CamelCase version and remove invalid chars
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 4b935c51b..faac0b12f 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -3,6 +3,8 @@ This module contains some assorted functions used in tests
"""
from __future__ import absolute_import
+from posixpath import split
+import asyncio
import os
from importlib import import_module
@@ -32,6 +34,7 @@ def skip_if_no_boto():
except NotConfigured as e:
raise SkipTest(e)
+
def get_s3_content_and_delete(bucket, path, with_key=False):
""" Get content from s3 key, and delete key afterwards.
"""
@@ -51,6 +54,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False):
bucket.delete_key(path)
return (content, key) if with_key else content
+
def get_gcs_content_and_delete(bucket, path):
from google.cloud import storage
client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
@@ -61,6 +65,27 @@ def get_gcs_content_and_delete(bucket, path):
bucket.delete_blob(path)
return content, acl, blob
+
+def get_ftp_content_and_delete(
+ path, host, port, username,
+ password, use_active_mode=False):
+ from ftplib import FTP
+ ftp = FTP()
+ ftp.connect(host, port)
+ ftp.login(username, password)
+ if use_active_mode:
+ ftp.set_pasv(False)
+ ftp_data = []
+
+ def buffer_data(data):
+ ftp_data.append(data)
+ ftp.retrbinary('RETR %s' % path, buffer_data)
+ dirname, filename = split(path)
+ ftp.cwd(dirname)
+ ftp.delete(filename)
+ return "".join(ftp_data)
+
+
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
@@ -72,12 +97,14 @@ def get_crawler(spidercls=None, settings_dict=None):
runner = CrawlerRunner(settings_dict)
return runner.create_crawler(spidercls or Spider)
+
def get_pythonpath():
"""Return a PYTHONPATH suitable to use in processes so that they find this
installation of Scrapy"""
scrapy_path = import_module('scrapy').__path__[0]
return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
+
def get_testenv():
"""Return a OS environment dict suitable to fork processes that need to import
this installation of Scrapy, instead of a system installed one.
@@ -86,8 +113,16 @@ def get_testenv():
env['PYTHONPATH'] = get_pythonpath()
return env
+
def assert_samelines(testcase, text1, text2, msg=None):
"""Asserts text1 and text2 have the same lines, ignoring differences in
line endings between platforms
"""
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
+
+
+def get_from_asyncio_queue(value):
+ q = asyncio.Queue()
+ getter = q.get()
+ q.put_nowait(value)
+ return getter
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index f268e91ff..0f15cf60a 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
import sys
import os
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index e50a989b3..6f5c21624 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,5 +1,4 @@
-from __future__ import print_function
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
from twisted.internet import reactor
from twisted.web import server, resource, static, util
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index eed14c5a1..4842b95df 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -9,12 +9,10 @@ and no performance penalty at all when disabled (as object_ref becomes just an
alias to object in that case).
"""
-from __future__ import print_function
import weakref
from time import time
from operator import itemgetter
from collections import defaultdict
-import six
NoneType = type(None)
@@ -37,13 +35,13 @@ def format_live_refs(ignore=NoneType):
"""Return a tabular representation of tracked objects"""
s = "Live References\n\n"
now = time()
- for cls, wdict in sorted(six.iteritems(live_refs),
+ for cls, wdict in sorted(live_refs.items(),
key=lambda x: x[0].__name__):
if not wdict:
continue
if issubclass(cls, ignore):
continue
- oldest = min(six.itervalues(wdict))
+ oldest = min(wdict.values())
s += "%-30s %6d oldest: %ds ago\n" % (
cls.__name__, len(wdict), now - oldest
)
@@ -57,15 +55,15 @@ def print_live_refs(*a, **kw):
def get_oldest(class_name):
"""Get the oldest object for a specific class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
if not wdict:
break
- return min(six.iteritems(wdict), key=itemgetter(1))[0]
+ return min(wdict.items(), key=itemgetter(1))[0]
def iter_all(class_name):
"""Iterate over all objects of the same class by its class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
- return six.iterkeys(wdict)
+ return wdict.keys()
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 657c53815..c9abb12d5 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,12 +7,12 @@ to the w3lib.url module. Always import those from there instead.
"""
import posixpath
import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
+from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
# scrapy.utils.url was moved to w3lib.url and import * ensures this
# move doesn't break old code
from w3lib.url import *
-from w3lib.url import _safe_chars, _unquotepath
+from w3lib.url import _safe_chars, _unquotepath # noqa: F401
from scrapy.utils.python import to_unicode
@@ -109,12 +109,12 @@ def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=
"""Strip URL string from some of its components:
- - `strip_credentials` removes "user:password@"
- - `strip_default_port` removes ":80" (resp. ":443", ":21")
+ - ``strip_credentials`` removes "user:password@"
+ - ``strip_default_port`` removes ":80" (resp. ":443", ":21")
from http:// (resp. https://, ftp://) URLs
- - `origin_only` replaces path component with "/", also dropping
+ - ``origin_only`` replaces path component with "/", also dropping
query and fragment components ; it also strips credentials
- - `strip_fragment` drops any #fragment component
+ - ``strip_fragment`` drops any #fragment component
"""
parsed_url = urlparse(url)
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 58c7aef85..b0737d3d5 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -1,6 +1,7 @@
import platform
import sys
+import cryptography
import cssselect
import lxml.etree
import parsel
@@ -8,20 +9,12 @@ import twisted
import w3lib
import scrapy
+from scrapy.utils.ssl import get_openssl_version
def scrapy_components_versions():
lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
- try:
- w3lib_version = w3lib.__version__
- except AttributeError:
- w3lib_version = "<1.14.3"
- try:
- import cryptography
- cryptography_version = cryptography.__version__
- except ImportError:
- cryptography_version = "unknown"
return [
("Scrapy", scrapy.__version__),
@@ -29,22 +22,10 @@ def scrapy_components_versions():
("libxml2", libxml2_version),
("cssselect", cssselect.__version__),
("parsel", parsel.__version__),
- ("w3lib", w3lib_version),
+ ("w3lib", w3lib.__version__),
("Twisted", twisted.version.short()),
("Python", sys.version.replace("\n", "- ")),
- ("pyOpenSSL", _get_openssl_version()),
- ("cryptography", cryptography_version),
- ("Platform", platform.platform()),
+ ("pyOpenSSL", get_openssl_version()),
+ ("cryptography", cryptography.__version__),
+ ("Platform", platform.platform()),
]
-
-
-def _get_openssl_version():
- try:
- import OpenSSL
- openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)\
- .decode('ascii', errors='replace')
- # pyOpenSSL 0.12 does not expose openssl version
- except AttributeError:
- openssl = 'Unknown OpenSSL version'
-
- return '{} ({})'.format(OpenSSL.version.__version__, openssl)
diff --git a/scrapy/xlib/__init__.py b/scrapy/xlib/__init__.py
deleted file mode 100644
index 11f022087..000000000
--- a/scrapy/xlib/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-"""This package contains some third party modules that are distributed along
-with Scrapy"""
diff --git a/scrapy/xlib/pydispatch.py b/scrapy/xlib/pydispatch.py
deleted file mode 100644
index 5ffeaf579..000000000
--- a/scrapy/xlib/pydispatch.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import absolute_import
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-from pydispatch import (
- dispatcher,
- errors,
- robust,
- robustapply,
- saferef,
-)
-
-warnings.warn("Importing from scrapy.xlib.pydispatch is deprecated and will"
- " no longer be supported in future Scrapy versions."
- " If you just want to connect signals use the from_crawler class method,"
- " otherwise import pydispatch directly if needed."
- " See: https://github.com/scrapy/scrapy/issues/1762",
- ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/xlib/tx.py b/scrapy/xlib/tx.py
deleted file mode 100644
index 0d94307b7..000000000
--- a/scrapy/xlib/tx.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from __future__ import absolute_import
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-from twisted.web import client
-from twisted.internet import endpoints
-
-Agent = client.Agent # since < 11.1
-ProxyAgent = client.ProxyAgent # since 11.1
-ResponseDone = client.ResponseDone # since 11.1
-ResponseFailed = client.ResponseFailed # since 11.1
-HTTPConnectionPool = client.HTTPConnectionPool # since 12.1
-TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1
-
-warnings.warn("Importing from scrapy.xlib.tx is deprecated and will"
- " no longer be supported in future Scrapy versions."
- " Update your code to import from twisted proper.",
- ScrapyDeprecationWarning, stacklevel=2)
diff --git a/sep/sep-001.rst b/sep/sep-001.rst
index 3766f38fc..00226283f 100644
--- a/sep/sep-001.rst
+++ b/sep/sep-001.rst
@@ -61,7 +61,7 @@ ItemForm
--------
Pros:
-- same API used for Items (see https://doc.scrapy.org/en/latest/topics/items.html)
+- same API used for Items (see https://docs.scrapy.org/en/latest/topics/items.html)
- some people consider setitem API more elegant than methods API
Cons:
@@ -254,8 +254,8 @@ ItemForm
#!python
class MySiteForm(ItemForm):
- witdth = adaptor(ItemForm.witdh, default_unit='cm')
- volume = adaptor(ItemForm.witdh, default_unit='lt')
+ width = adaptor(ItemForm.width, default_unit='cm')
+ volume = adaptor(ItemForm.width, default_unit='lt')
ia['width'] = x.x('//p[@class="width"]')
ia['volume'] = x.x('//p[@class="volume"]')
diff --git a/sep/sep-004.rst b/sep/sep-004.rst
index 69edfa136..05b0eb99c 100644
--- a/sep/sep-004.rst
+++ b/sep/sep-004.rst
@@ -53,7 +53,7 @@ Here's a simple proof-of-concept code of such script:
# ... do something more interesting with scraped_items ...
The behaviour of the Scrapy crawler would be controller by the Scrapy settings,
-naturally, just like any typical scrapy project. But the default settings
+naturally, just like any typical Scrapy project. But the default settings
should be sufficient so as to not require adding any specific setting. But, at
the same time, you could do it if you need to, say, for specifying a custom
middleware.
diff --git a/sep/sep-006.rst b/sep/sep-006.rst
index 7425c0930..eb362e945 100644
--- a/sep/sep-006.rst
+++ b/sep/sep-006.rst
@@ -10,13 +10,14 @@ Status Obsolete (discarded)
SEP-006: Rename of Selectors to Extractors
==========================================
-This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and their `x` method.
+This SEP proposes a more meaningful naming of XPathSelectors or "Selectors" and
+their ``x`` method.
Motivation
==========
When you use Selectors in Scrapy, your final goal is to "extract" the data that
-you've selected, as the [https://doc.scrapy.org/en/latest/topics/selectors.html
+you've selected, as the [https://docs.scrapy.org/en/latest/topics/selectors.html
XPath Selectors documentation] says (bolding by me):
When you’re scraping web pages, the most common task you need to perform is
@@ -57,7 +58,7 @@ Additional changes
As the name of the method for performing selection (the ``x`` method) is not
descriptive nor mnemotechnic enough and clearly clashes with ``extract`` method
(x sounds like a short for extract in english), we propose to rename it to
-`select`, `sel` (is shortness if required), or `xpath` after `lxml's
+``select``, ``sel`` (is shortness if required), or ``xpath`` after `lxml's
<http://lxml.de/xpathxslt.html>`_ ``xpath`` method.
Bonus (ItemBuilder)
@@ -71,5 +72,5 @@ webpage or set of pages.
References
==========
- 1. XPath Selectors (https://doc.scrapy.org/topics/selectors.html)
+ 1. XPath Selectors (https://docs.scrapy.org/topics/selectors.html)
2. XPath and XSLT with lxml (http://lxml.de/xpathxslt.html)
diff --git a/sep/sep-009.rst b/sep/sep-009.rst
index 232a536a8..da87fa9aa 100644
--- a/sep/sep-009.rst
+++ b/sep/sep-009.rst
@@ -38,7 +38,7 @@ singletons members of that object, as explained below:
``scrapy.core.manager.ExecutionManager``) - instantiated with a ``Settings``
object
- - **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the constructor)
+ - **crawler.settings**: ``scrapy.conf.Settings`` instance (passed in the ``__init__`` method)
- **crawler.extensions**: ``scrapy.extension.ExtensionManager`` instance
- **crawler.engine**: ``scrapy.core.engine.ExecutionEngine`` instance
- ``crawler.engine.scheduler``
@@ -55,7 +55,7 @@ singletons members of that object, as explained below:
``STATS_CLASS`` setting)
- **crawler.log**: Logger class with methods replacing the current
``scrapy.log`` functions. Logging would be started (if enabled) on
- ``Crawler`` constructor, so no log starting functions are required.
+ ``Crawler`` instantiation, so no log starting functions are required.
- ``crawler.log.msg``
- **crawler.signals**: signal handling
@@ -69,12 +69,12 @@ Required code changes after singletons removal
==============================================
All components (extensions, middlewares, etc) will receive this ``Crawler``
-object in their constructors, and this will be the only mechanism for accessing
+object in their ``__init__`` methods, and this will be the only mechanism for accessing
any other components (as opposed to importing each singleton from their
respective module). This will also serve to stabilize the core API, something
which we haven't documented so far (partly because of this).
-So, for a typical middleware constructor code, instead of this:
+So, for a typical middleware ``__init__`` method code, instead of this:
::
@@ -125,13 +125,13 @@ Open issues to resolve
- Should we pass ``Settings`` object to ``ScrapyCommand.add_options()``?
- How should spiders access settings?
- - Option 1. Pass ``Crawler`` object to spider constructors too
+ - Option 1. Pass ``Crawler`` object to spider ``__init__`` methods too
- pro: one way to access all components (settings and signals being the
most relevant to spiders)
- con?: spider code can access (and control) any crawler component -
since we don't want to support spiders messing with the crawler (write
an extension or spider middleware if you need that)
- - Option 2. Pass ``Settings`` object to spider constructors, which would
+ - Option 2. Pass ``Settings`` object to spider ``__init__`` methods, which would
then be accessed through ``self.settings``, like logging which is accessed
through ``self.log``
diff --git a/sep/sep-018.rst b/sep/sep-018.rst
index aca7ac342..fe707923a 100644
--- a/sep/sep-018.rst
+++ b/sep/sep-018.rst
@@ -211,7 +211,7 @@ spider methods on each event such as:
- call additional spider middlewares defined in the ``Spider.middlewares``
attribute
- call ``Spider.next_request()`` and ``Spider.start_requests()`` on
- ``next_request()`` middleware method (this would implicitly support backwards
+ ``next_request()`` middleware method (this would implicitly support backward
compatibility)
Differences with Spider middleware v1
diff --git a/sep/sep-019.rst b/sep/sep-019.rst
index 9fbf6a223..84f3a96c3 100644
--- a/sep/sep-019.rst
+++ b/sep/sep-019.rst
@@ -185,7 +185,7 @@ These ideas translate to the following changes on the ``SpiderManager`` class:
will return a spider class, not an instance. It's basically a ``__get__``
to ``self._spiders``.
-- All remaining functions should be deprecated or remove accordantly, since a
+- All remaining functions should be deprecated or remove accordingly, since a
crawler reference is no longer needed.
- New helper ``get_spider_manager_class_from_scrapycfg`` in
diff --git a/setup.py b/setup.py
index c37919cda..85d797f88 100644
--- a/setup.py
+++ b/setup.py
@@ -50,30 +50,31 @@ setup(
'License :: OSI Approved :: BSD License',
'Operating System :: OS Independent',
'Programming Language :: Python',
- 'Programming Language :: Python :: 2',
- 'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
+ 'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: Implementation :: CPython',
'Programming Language :: Python :: Implementation :: PyPy',
'Topic :: Internet :: WWW/HTTP',
'Topic :: Software Development :: Libraries :: Application Frameworks',
'Topic :: Software Development :: Libraries :: Python Modules',
],
- python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+ python_requires='>=3.5',
install_requires=[
- 'Twisted>=13.1.0',
- 'w3lib>=1.17.0',
- 'queuelib',
- 'lxml',
- 'pyOpenSSL',
- 'cssselect>=0.9',
- 'six>=1.5.2',
- 'parsel>=1.4',
+ 'Twisted>=17.9.0',
+ 'cryptography>=2.0',
+ 'cssselect>=0.9.1',
+ 'lxml>=3.5.0',
+ 'parsel>=1.5.0',
'PyDispatcher>=2.0.5',
- 'service_identity',
+ 'pyOpenSSL>=16.2.0',
+ 'queuelib>=1.4.2',
+ 'service_identity>=16.0.0',
+ 'w3lib>=1.17.0',
+ 'zope.interface>=4.1.3',
+ 'protego>=0.1.15',
],
extras_require=extras_require,
)
diff --git a/tests/CrawlerProcess/alternative_name_resolver.py b/tests/CrawlerProcess/alternative_name_resolver.py
new file mode 100644
index 000000000..2c466da04
--- /dev/null
+++ b/tests/CrawlerProcess/alternative_name_resolver.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+ name = "ipv6_spider"
+ start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={
+ "RETRY_ENABLED": False,
+ "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+})
+process.crawl(IPv6Spider)
+process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
new file mode 100644
index 000000000..db1b75931
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py
@@ -0,0 +1,17 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={
+ 'ASYNCIO_REACTOR': True,
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
new file mode 100644
index 000000000..cec3c9c25
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -0,0 +1,22 @@
+import asyncio
+
+from twisted.internet import asyncioreactor
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={
+ 'ASYNCIO_REACTOR': True,
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py
new file mode 100644
index 000000000..60d91b68b
--- /dev/null
+++ b/tests/CrawlerProcess/default_name_resolver.py
@@ -0,0 +1,12 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+ name = "ipv6_spider"
+ start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={"RETRY_ENABLED": False})
+process.crawl(IPv6Spider)
+process.start()
diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py
new file mode 100644
index 000000000..5f6f1ae30
--- /dev/null
+++ b/tests/CrawlerProcess/simple.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/__init__.py b/tests/__init__.py
index 55b1ecde8..12ce79fa9 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,7 +1,7 @@
"""
tests: this package contains all Scrapy unittests
-see https://doc.scrapy.org/en/latest/contributing.html#running-tests
+see https://docs.scrapy.org/en/latest/contributing.html#running-tests
"""
import os
@@ -21,11 +21,6 @@ if 'COV_CORE_CONFIG' in os.environ:
os.environ['COV_CORE_CONFIG'] = os.path.join(_sourceroot,
os.environ['COV_CORE_CONFIG'])
-try:
- import unittest.mock as mock
-except ImportError:
- import mock
-
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'sample_data')
diff --git a/tests/constraints.txt b/tests/constraints.txt
index 3bc30de15..5655ac2d3 100644
--- a/tests/constraints.txt
+++ b/tests/constraints.txt
@@ -1 +1 @@
-Twisted!=18.4.0
+Twisted!=18.4.0
\ No newline at end of file
diff --git a/tests/py3-ignores.txt b/tests/ignores.txt
similarity index 74%
rename from tests/py3-ignores.txt
rename to tests/ignores.txt
index 313e74ec9..45cf6fb92 100644
--- a/tests/py3-ignores.txt
+++ b/tests/ignores.txt
@@ -1,6 +1,3 @@
-tests/test_linkextractors_deprecated.py
-tests/test_proxy_connect.py
-
scrapy/linkextractors/sgml.py
scrapy/linkextractors/regex.py
scrapy/linkextractors/htmlparser.py
diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py
index 40d9293b2..75c74daf5 100644
--- a/tests/mocks/dummydbm.py
+++ b/tests/mocks/dummydbm.py
@@ -13,10 +13,11 @@ error = KeyError
_DATABASES = collections.defaultdict(DummyDB)
+
def open(file, flag='r', mode=0o666):
"""Open or create a dummy database compatible.
- Arguments `flag` and `mode` are ignored.
+ Arguments ``flag`` and ``mode`` are ignored.
"""
# return same instance for same file argument
return _DATABASES[file]
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 98723846e..a45277db9 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,8 +1,11 @@
-from __future__ import print_function
-import sys, time, random, os, json
-from six.moves.urllib.parse import urlencode
+import json
+import os
+import random
+import sys
from subprocess import Popen, PIPE
+from urllib.parse import urlencode
+from OpenSSL import SSL
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.web.static import File
@@ -13,8 +16,8 @@ from twisted.web.util import redirectTo
from twisted.internet import reactor, ssl
from twisted.internet.task import deferLater
-
from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
def getarg(request, name, default=None, type=None):
@@ -161,6 +164,12 @@ class Drop(Partial):
request.finish()
+class ArbitraryLengthPayloadResource(LeafResource):
+
+ def render(self, request):
+ return request.content.read()
+
+
class Root(Resource):
def __init__(self):
@@ -174,10 +183,11 @@ class Root(Resource):
self.putChild(b"echo", Echo())
self.putChild(b"payload", PayloadResource())
self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+ self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
try:
from tests import tests_datadir
self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
- except:
+ except Exception:
pass
self.putChild(b"redirect-to", RedirectTo())
@@ -192,35 +202,55 @@ class MockServer():
def __enter__(self):
from scrapy.utils.test import get_testenv
+
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
stdout=PIPE, env=get_testenv())
- self.proc.stdout.readline()
+ http_address = self.proc.stdout.readline().strip().decode('ascii')
+ https_address = self.proc.stdout.readline().strip().decode('ascii')
+
+ self.http_address = http_address
+ self.https_address = https_address
+
return self
def __exit__(self, exc_type, exc_value, traceback):
self.proc.kill()
- self.proc.wait()
- time.sleep(0.2)
+ self.proc.communicate()
+
+ def url(self, path, is_secure=False):
+ host = self.http_address.replace('0.0.0.0', '127.0.0.1')
+ if is_secure:
+ host = self.https_address
+ return host + path
-def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'):
- return ssl.DefaultOpenSSLContextFactory(
+def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
+ factory = ssl.DefaultOpenSSLContextFactory(
os.path.join(os.path.dirname(__file__), keyfile),
os.path.join(os.path.dirname(__file__), certfile),
)
+ if cipher_string:
+ ctx = factory.getContext()
+ # disabling TLS1.2+ because it unconditionally enables some strong ciphers
+ ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3)
+ ctx.set_cipher_list(to_bytes(cipher_string))
+ return factory
if __name__ == "__main__":
root = Root()
factory = Site(root)
- httpPort = reactor.listenTCP(8998, factory)
+ httpPort = reactor.listenTCP(0, factory)
contextFactory = ssl_context_factory()
- httpsPort = reactor.listenSSL(8999, factory, contextFactory)
+ httpsPort = reactor.listenSSL(0, factory, contextFactory)
def print_listening():
httpHost = httpPort.getHost()
httpsHost = httpsPort.getHost()
- print("Mock server running at http://%s:%d and https://%s:%d" % (
- httpHost.host, httpHost.port, httpsHost.host, httpsHost.port))
+ httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
+ httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
+ print(httpAddress)
+ print(httpsAddress)
+
reactor.callWhenRunning(print_listening)
reactor.run()
diff --git a/tests/pipelines.py b/tests/pipelines.py
index ddfbc7a99..d7d3b5259 100644
--- a/tests/pipelines.py
+++ b/tests/pipelines.py
@@ -2,6 +2,7 @@
Some pipelines used for testing
"""
+
class ZeroDivisionErrorPipeline(object):
def open_spider(self, spider):
@@ -9,3 +10,9 @@ class ZeroDivisionErrorPipeline(object):
def process_item(self, item, spider):
return item
+
+
+class ProcessWithZeroDivisionErrorPipiline(object):
+
+ def process_item(self, item, spider):
+ 1/0
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 51a25f5e5..d97c4b8ee 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,11 +1,16 @@
-pytest==2.9.2
-pytest-twisted
-pytest-cov==2.2.1
-testfixtures
+# Tests requirements
jmespath
-leveldb
-botocore
+mitmproxy; python_version >= '3.6'
+mitmproxy<4.0.0; python_version < '3.6'
+pytest
+pytest-cov
+pytest-twisted >= 1.11
+pytest-xdist
+sybil
+testfixtures
+
# optional for shell wrapper tests
bpython
-ipython
brotlipy
+ipython
+pywin32; sys_platform == "win32"
diff --git a/tests/requirements.txt b/tests/requirements.txt
deleted file mode 100644
index c1576a2e7..000000000
--- a/tests/requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# Tests requirements
-mock
-mitmproxy==0.10.1
-netlib==0.10.1
-pytest==2.9.2
-pytest-twisted
-pytest-cov==2.2.1
-jmespath
-brotlipy
-testfixtures
-# optional for shell wrapper tests
-bpython
-ipython
diff --git a/tests/sample_data/link_extractor/sgml_linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
similarity index 100%
rename from tests/sample_data/link_extractor/sgml_linkextractor.html
rename to tests/sample_data/link_extractor/linkextractor.html
diff --git a/tests/sample_data/link_extractor/linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html
new file mode 100644
index 000000000..0b01cede8
--- /dev/null
+++ b/tests/sample_data/link_extractor/linkextractor_no_href.html
@@ -0,0 +1,25 @@
+<html>
+ <head>
+ <base href='http://example.com' />
+ <title>Sample page with anchor tags containing no href attribute, to test the TextResponse.follow_all method
+
+
+
+
+
“The world as we have created it is a process of our
+ thinking. It cannot be changed without changing our thinking.”
+
+ by Albert Einstein
+ (about)
+
+
+