Merge branch 'master' into duplicate_key

.bandit.yml

@@ -0,0 +1,18 @@
+skips:
+- B101
+- B105
+- B301
+- B303
+- B306
+- B307
+- B311
+- B320
+- B321
+- B402  # https://github.com/scrapy/scrapy/issues/4180
+- B403
+- B404
+- B406
+- B410
+- B503
+- B603
+- B605

.bumpversion.cfg

@@ -1,8 +1,7 @@
 [bumpversion]
-current_version = 1.5.0
+current_version = 2.0.0
 commit = True
 tag = True
 tag_name = {new_version}
 
 [bumpversion:file:scrapy/VERSION]
-

.coveragerc

@@ -3,6 +3,3 @@ branch = true
 include = scrapy/*
 omit =
   tests/*
-  scrapy/xlib/*
-  scrapy/conf.py
-  scrapy/log.py

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Bug report
+about: Report a problem to help us improve
+---
+
+<!--
+
+Thanks for taking an interest in Scrapy!
+
+If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
+The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
+
+Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
+
+The following is a suggested template to structure your issue, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#reporting-bugs
+
+-->
+
+### Description
+
+[Description of the issue]
+
+### Steps to Reproduce
+
+1. [First Step]
+2. [Second Step]
+3. [and so on...]
+
+**Expected behavior:** [What you expect to happen]
+
+**Actual behavior:** [What actually happens]
+
+**Reproduces how often:** [What percentage of the time does it reproduce?]
+
+### Versions
+
+Please paste here the output of executing `scrapy version --verbose` in the command line.
+
+### Additional context
+
+Any additional information, configuration, data or output from commands that might be necessary to reproduce or understand the issue. Please try not to include screenshots of code or the command line, paste the contents as text instead. You can use [GitHub Flavored Markdown](https://help.github.com/en/articles/creating-and-highlighting-code-blocks) to make the text look better.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,33 @@
+---
+name: Feature request
+about: Suggest an idea for an enhancement or new feature
+---
+
+<!--
+
+Thanks for taking an interest in Scrapy!
+
+If you have a question that starts with "How to...", please see the Scrapy Community page: https://scrapy.org/community/.
+The GitHub issue tracker's purpose is to deal with bug reports and feature requests for the project itself.
+
+Keep in mind that by filing an issue, you are expected to comply with Scrapy's Code of Conduct, including treating everyone with respect: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
+
+The following is a suggested template to structure your pull request, you can find more guidelines at https://doc.scrapy.org/en/latest/contributing.html#writing-patches and https://doc.scrapy.org/en/latest/contributing.html#submitting-patches
+
+-->
+
+## Summary
+
+One paragraph explanation of the feature.
+
+## Motivation
+
+Why are we doing this? What use cases does it support? What is the expected outcome?
+
+## Describe alternatives you've considered
+
+A clear and concise description of the alternative solutions you've considered. Be sure to explain why Scrapy's existing customizability isn't suitable for this feature.
+
+## Additional context
+
+Any additional information about the feature request here.

.gitignore

@@ -15,7 +15,6 @@ htmlcov/
 .pytest_cache/
 .coverage.*
 .cache/
-.pytest_cache/
 
 # Windows
 Thumbs.db

.readthedocs.yml

@@ -0,0 +1,11 @@
+version: 2
+sphinx:
+  configuration: docs/conf.py
+  fail_on_warning: true
+python:
+  # For available versions, see:
+  # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-image
+  version: 3.7  # Keep in sync with .travis.yml
+  install:
+    - requirements: docs/requirements.txt
+    - path: .

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+dist: xenial
 branches:
   only:
     - master
@@ -6,35 +7,31 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=jessie
-    - python: 2.7
-      env: TOXENV=pypy
-    - python: 2.7
-      env: TOXENV=pypy3
-    - python: 3.4
-      env: TOXENV=py34
-    - python: 3.5
-      env: TOXENV=py35
-    - python: 3.6
-      env: TOXENV=py36
-    - python: 3.7
-      env: TOXENV=py37
-      dist: xenial
-      sudo: true
-    - python: 3.6
-      env: TOXENV=docs
+    - env: TOXENV=security
+      python: 3.8
+    - env: TOXENV=flake8
+      python: 3.8
+    - env: TOXENV=pypy3
+    - env: TOXENV=py35
+      python: 3.5
+    - env: TOXENV=pinned
+      python: 3.5
+    - env: TOXENV=py35-asyncio
+      python: 3.5.2
+    - env: TOXENV=py36
+      python: 3.6
+    - env: TOXENV=py37
+      python: 3.7
+    - env: TOXENV=py38
+      python: 3.8
+    - env: TOXENV=extra-deps
+      python: 3.8
+    - env: TOXENV=py38-asyncio
+      python: 3.8
+    - env: TOXENV=docs
+      python: 3.7  # Keep in sync with .readthedocs.yml
 install:
   - |
-      if [ "$TOXENV" = "pypy" ]; then
-        export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
-        wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
-        tar -jxf ${PYPY_VERSION}.tar.bz2
-        virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
-        source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-      fi
       if [ "$TOXENV" = "pypy3" ]; then
         export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
         wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
@@ -65,4 +62,4 @@ deploy:
   on:
     tags: true
     repo: scrapy/scrapy
-    condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
+    condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

CODE_OF_CONDUCT.md

@@ -68,7 +68,7 @@ members of the project's leadership.
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
+available at [http://contributor-covenant.org/version/1/4][version].
 
 [homepage]: http://contributor-covenant.org
 [version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -1,5 +1,5 @@
 The guidelines for contributing are available here:
-https://doc.scrapy.org/en/master/contributing.html
+https://docs.scrapy.org/en/master/contributing.html
 
 Please do not abuse the issue tracker for support questions.
 If your issue topic can be rephrased to "How to ...?", please use the

INSTALL

@@ -1,4 +1,4 @@
 For information about installing Scrapy see:
 
 * docs/intro/install.rst (local file)
-* https://doc.scrapy.org/en/latest/intro/install.html (online version)
+* https://docs.scrapy.org/en/latest/intro/install.html (online version)

Makefile.buildbot

@@ -1,24 +0,0 @@
-TRIAL := $(shell which trial)
-BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
-export PYTHONPATH=$(PWD)
-
-test:
-	coverage run --branch $(TRIAL) --reporter=text tests
-	rm -rf htmlcov && coverage html
-	-s3cmd sync -P htmlcov/ s3://static.scrapy.org/coverage-scrapy-$(BRANCH)/
-
-build:
-	git describe --tags --match '[0-9]*' |sed 's/-/.post/;s/-g/+g/' >scrapy/VERSION
-	debchange -m -D unstable --force-distribution -v \
-	  $$(python setup.py --version |sed -r 's/([0-9]+.[0-9]+.[0-9]+)(a|b|rc|dev)([0-9]*)/\1~\2\3/')-$$(date +%s) \
-	  "Automatic build"
-	debuild -us -uc -b
-
-clean:
-	git checkout debian scrapy/VERSION
-	git clean -dfq
-
-pypi:
-	umask 0022 &&  chmod -R a+rX . && python setup.py sdist upload
-
-.PHONY: clean test build

README.rst

@@ -34,14 +34,14 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
 crawl websites and extract structured data from their pages. It can be used for
 a wide range of purposes, from data mining to monitoring and automated testing.
 
-For more information including a list of features check the Scrapy homepage at:
-https://scrapy.org
+Check the Scrapy homepage at https://scrapy.org for more information,
+including a list of features.
 
 Requirements
 ============
 
-* Python 2.7 or Python 3.4+
-* Works on Linux, Windows, Mac OSX, BSD
+* Python 3.5+
+* Works on Linux, Windows, macOS, BSD
 
 Install
 =======
@@ -50,29 +50,29 @@ The quick way::
 
     pip install scrapy
 
-For more details see the install section in the documentation:
-https://doc.scrapy.org/en/latest/intro/install.html
+See the install section in the documentation at
+https://docs.scrapy.org/en/latest/intro/install.html for more details.
 
 Documentation
 =============
 
-Documentation is available online at https://doc.scrapy.org/ and in the ``docs``
+Documentation is available online at https://docs.scrapy.org/ and in the ``docs``
 directory.
 
 Releases
 ========
 
-You can find release notes at https://doc.scrapy.org/en/latest/news.html
+You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
 
 Community (blog, twitter, mail list, IRC)
 =========================================
 
-See https://scrapy.org/community/
+See https://scrapy.org/community/ for details.
 
 Contributing
 ============
 
-See https://doc.scrapy.org/en/master/contributing.html
+See https://docs.scrapy.org/en/master/contributing.html for details.
 
 Code of Conduct
 ---------------
@@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
 Companies using Scrapy
 ======================
 
-See https://scrapy.org/companies/
+See https://scrapy.org/companies/ for a list.
 
 Commercial Support
 ==================
 
-See https://scrapy.org/support/
+See https://scrapy.org/support/ for details.

appveyor.yml

@@ -12,7 +12,8 @@ branches:
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"
+  - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
+  - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
   - "pip install -U tox"
 
 build: false

artwork/README.rst

@@ -1,5 +1,4 @@
-:orphan:
-
+==============
 Scrapy artwork
 ==============
 

conftest.py

@@ -1,35 +1,53 @@
-import glob
-import six
+from pathlib import Path
+
 import pytest
-from twisted import version as twisted_version
 
 
 def _py_files(folder):
-    return glob.glob(folder + "/*.py") + glob.glob(folder + "/*/*.py")
+    return (str(p) for p in Path(folder).rglob('*.py'))
 
 
 collect_ignore = [
-    # deprecated or moved modules
-    "scrapy/conf.py",
-    "scrapy/log.py",
-
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
-
+    # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+    *_py_files("tests/CrawlerProcess"),
+    # Py36-only parts of respective tests
+    *_py_files("tests/py36"),
 ]
 
-if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, 5, 0):
-    collect_ignore += _py_files("scrapy/xlib/tx")
-
-
-if six.PY3:
-    for line in open('tests/py3-ignores.txt'):
-        file_path = line.strip()
-        if file_path and file_path[0] != '#':
-            collect_ignore.append(file_path)
+for line in open('tests/ignores.txt'):
+    file_path = line.strip()
+    if file_path and file_path[0] != '#':
+        collect_ignore.append(file_path)
 
 
 @pytest.fixture()
 def chdir(tmpdir):
     """Change to pytest-provided temporary directory"""
     tmpdir.chdir()
+
+
+def pytest_collection_modifyitems(session, config, items):
+    # Avoid executing tests when executing `--flake8` flag (pytest-flake8)
+    try:
+        from pytest_flake8 import Flake8Item
+        if config.getoption('--flake8'):
+            items[:] = [item for item in items if isinstance(item, Flake8Item)]
+    except ImportError:
+        pass
+
+
+@pytest.fixture(scope='class')
+def reactor_pytest(request):
+    if not request.cls:
+        # doctests
+        return
+    request.cls.reactor_pytest = request.config.getoption("--reactor")
+    return request.cls.reactor_pytest
+
+
+@pytest.fixture(autouse=True)
+def only_asyncio(request, reactor_pytest):
+    if request.node.get_closest_marker('only_asyncio') and reactor_pytest != 'asyncio':
+        pytest.skip('This test is only run with --reactor=asyncio')

debian/changelog

@@ -1,5 +0,0 @@
-scrapy (0.11) unstable; urgency=low
-
-  * Initial release.
-
- -- Scrapinghub Team <info@scrapinghub.com>  Thu, 10 Jun 2010 17:24:02 -0300

debian/compat

@@ -1 +0,0 @@
-7

debian/control

@@ -1,20 +0,0 @@
-Source: scrapy
-Section: python
-Priority: optional
-Maintainer: Scrapinghub Team <info@scrapinghub.com>
-Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml, python-six (>=1.5.2)
-Standards-Version: 3.8.4
-Homepage: https://scrapy.org/
-
-Package: scrapy
-Architecture: all
-Depends: ${python:Depends}, python-lxml, python-twisted, python-openssl,
- python-w3lib (>= 1.8.0), python-queuelib, python-cssselect (>= 0.9), python-six (>=1.5.2)
-Recommends: python-setuptools
-Conflicts: python-scrapy, scrapy-0.25
-Provides: python-scrapy, scrapy-0.25
-Description: Python web crawling and web scraping framework
- Scrapy is a fast high-level web crawling and web scraping framework,
- used to crawl websites and extract structured data from their pages.
- It can be used for a wide range of purposes, from data mining to
- monitoring and automated testing.

debian/copyright

@@ -1,40 +0,0 @@
-This package was debianized by the Scrapinghub team <info@scrapinghub.com>.
-
-It was downloaded from https://scrapy.org
-
-Upstream Author: Scrapy Developers
-
-Copyright: 2007-2013 Scrapy Developers
-
-License: bsd
-
-Copyright (c) Scrapy developers.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-    1. Redistributions of source code must retain the above copyright notice,
-       this list of conditions and the following disclaimer.
-
-    2. Redistributions in binary form must reproduce the above copyright
-       notice, this list of conditions and the following disclaimer in the
-       documentation and/or other materials provided with the distribution.
-
-    3. Neither the name of Scrapy nor the names of its contributors may be used
-       to endorse or promote products derived from this software without
-       specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-The Debian packaging is (C) 2010-2013, Scrapinghub <info@scrapinghub.com> and
-is licensed under the BSD, see `/usr/share/common-licenses/BSD'.

debian/pyversions

@@ -1 +0,0 @@
-2.7

debian/rules

@@ -1,5 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-%:
-	dh $@

debian/scrapy.docs

@@ -1,2 +0,0 @@
-README.rst
-AUTHORS

debian/scrapy.install

@@ -1,2 +0,0 @@
-extras/scrapy_bash_completion etc/bash_completion.d/
-extras/scrapy_zsh_completion /usr/share/zsh/vendor-completions/_scrapy

debian/scrapy.lintian-overrides

@@ -1,2 +0,0 @@
-new-package-should-close-itp-bug
-extra-license-file usr/share/pyshared/scrapy/xlib/pydispatch/license.txt

debian/scrapy.manpages

@@ -1 +0,0 @@
-extras/scrapy.1

docs/Makefile

@@ -82,6 +82,9 @@ pydoc-topics: build
 	@echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \
 	      "into the Lib/ directory"
 
+coverage: BUILDER = coverage
+coverage: build
+
 htmlview: html
 	 $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
 	 os.path.realpath('build/html/index.html'))"

docs/_tests/quotes.html

@@ -0,0 +1,281 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<title>Quotes to Scrape</title>
+    <link rel="stylesheet" href="/static/bootstrap.min.css">
+    <link rel="stylesheet" href="/static/main.css">
+</head>
+<body>
+    <div class="container">
+        <div class="row header-box">
+            <div class="col-md-8">
+                <h1>
+                    <a href="/" style="text-decoration: none">Quotes to Scrape</a>
+                </h1>
+            </div>
+            <div class="col-md-4">
+                <p>
+                
+                    <a href="/login">Login</a>
+                
+                </p>
+            </div>
+        </div>
+    
+
+<div class="row">
+    <div class="col-md-8">
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" /    > 
+            
+            <a class="tag" href="/tag/change/page/1/">change</a>
+            
+            <a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
+            
+            <a class="tag" href="/tag/thinking/page/1/">thinking</a>
+            
+            <a class="tag" href="/tag/world/page/1/">world</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
+        <span>by <small class="author" itemprop="author">J.K. Rowling</small>
+        <a href="/author/J-K-Rowling">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="abilities,choices" /    > 
+            
+            <a class="tag" href="/tag/abilities/page/1/">abilities</a>
+            
+            <a class="tag" href="/tag/choices/page/1/">choices</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" /    > 
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/live/page/1/">live</a>
+            
+            <a class="tag" href="/tag/miracle/page/1/">miracle</a>
+            
+            <a class="tag" href="/tag/miracles/page/1/">miracles</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
+        <span>by <small class="author" itemprop="author">Jane Austen</small>
+        <a href="/author/Jane-Austen">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" /    > 
+            
+            <a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
+            
+            <a class="tag" href="/tag/books/page/1/">books</a>
+            
+            <a class="tag" href="/tag/classic/page/1/">classic</a>
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
+        <span>by <small class="author" itemprop="author">Marilyn Monroe</small>
+        <a href="/author/Marilyn-Monroe">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" /    > 
+            
+            <a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="adulthood,success,value" /    > 
+            
+            <a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
+            
+            <a class="tag" href="/tag/success/page/1/">success</a>
+            
+            <a class="tag" href="/tag/value/page/1/">value</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
+        <span>by <small class="author" itemprop="author">André Gide</small>
+        <a href="/author/Andre-Gide">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="life,love" /    > 
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/love/page/1/">love</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
+        <span>by <small class="author" itemprop="author">Thomas A. Edison</small>
+        <a href="/author/Thomas-A-Edison">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" /    > 
+            
+            <a class="tag" href="/tag/edison/page/1/">edison</a>
+            
+            <a class="tag" href="/tag/failure/page/1/">failure</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
+        <span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
+        <a href="/author/Eleanor-Roosevelt">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" /    > 
+            
+            <a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
+        <span>by <small class="author" itemprop="author">Steve Martin</small>
+        <a href="/author/Steve-Martin">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="humor,obvious,simile" /    > 
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+            <a class="tag" href="/tag/obvious/page/1/">obvious</a>
+            
+            <a class="tag" href="/tag/simile/page/1/">simile</a>
+            
+        </div>
+    </div>
+
+    <nav>
+        <ul class="pager">
+            
+            
+            <li class="next">
+                <a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
+            </li>
+            
+        </ul>
+    </nav>
+    </div>
+    <div class="col-md-4 tags-box">
+        
+            <h2>Top Ten tags</h2>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
+            </span>
+            
+        
+    </div>
+</div>
+
+    </div>
+    <footer class="footer">
+        <div class="container">
+            <p class="text-muted">
+                Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
+            </p>
+            <p class="copyright">
+                Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
+            </p>
+        </div>
+    </footer>
+</body>
+</html>

docs/_tests/quotes1.html

@@ -0,0 +1,281 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<title>Quotes to Scrape</title>
+    <link rel="stylesheet" href="/static/bootstrap.min.css">
+    <link rel="stylesheet" href="/static/main.css">
+</head>
+<body>
+    <div class="container">
+        <div class="row header-box">
+            <div class="col-md-8">
+                <h1>
+                    <a href="/" style="text-decoration: none">Quotes to Scrape</a>
+                </h1>
+            </div>
+            <div class="col-md-4">
+                <p>
+                
+                    <a href="/login">Login</a>
+                
+                </p>
+            </div>
+        </div>
+    
+
+<div class="row">
+    <div class="col-md-8">
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="change,deep-thoughts,thinking,world" /    > 
+            
+            <a class="tag" href="/tag/change/page/1/">change</a>
+            
+            <a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
+            
+            <a class="tag" href="/tag/thinking/page/1/">thinking</a>
+            
+            <a class="tag" href="/tag/world/page/1/">world</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>
+        <span>by <small class="author" itemprop="author">J.K. Rowling</small>
+        <a href="/author/J-K-Rowling">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="abilities,choices" /    > 
+            
+            <a class="tag" href="/tag/abilities/page/1/">abilities</a>
+            
+            <a class="tag" href="/tag/choices/page/1/">choices</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="inspirational,life,live,miracle,miracles" /    > 
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/live/page/1/">live</a>
+            
+            <a class="tag" href="/tag/miracle/page/1/">miracle</a>
+            
+            <a class="tag" href="/tag/miracles/page/1/">miracles</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>
+        <span>by <small class="author" itemprop="author">Jane Austen</small>
+        <a href="/author/Jane-Austen">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="aliteracy,books,classic,humor" /    > 
+            
+            <a class="tag" href="/tag/aliteracy/page/1/">aliteracy</a>
+            
+            <a class="tag" href="/tag/books/page/1/">books</a>
+            
+            <a class="tag" href="/tag/classic/page/1/">classic</a>
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it&#39;s better to be absolutely ridiculous than absolutely boring.”</span>
+        <span>by <small class="author" itemprop="author">Marilyn Monroe</small>
+        <a href="/author/Marilyn-Monroe">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="be-yourself,inspirational" /    > 
+            
+            <a class="tag" href="/tag/be-yourself/page/1/">be-yourself</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>
+        <span>by <small class="author" itemprop="author">Albert Einstein</small>
+        <a href="/author/Albert-Einstein">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="adulthood,success,value" /    > 
+            
+            <a class="tag" href="/tag/adulthood/page/1/">adulthood</a>
+            
+            <a class="tag" href="/tag/success/page/1/">success</a>
+            
+            <a class="tag" href="/tag/value/page/1/">value</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>
+        <span>by <small class="author" itemprop="author">André Gide</small>
+        <a href="/author/Andre-Gide">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="life,love" /    > 
+            
+            <a class="tag" href="/tag/life/page/1/">life</a>
+            
+            <a class="tag" href="/tag/love/page/1/">love</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“I have not failed. I&#39;ve just found 10,000 ways that won&#39;t work.”</span>
+        <span>by <small class="author" itemprop="author">Thomas A. Edison</small>
+        <a href="/author/Thomas-A-Edison">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="edison,failure,inspirational,paraphrased" /    > 
+            
+            <a class="tag" href="/tag/edison/page/1/">edison</a>
+            
+            <a class="tag" href="/tag/failure/page/1/">failure</a>
+            
+            <a class="tag" href="/tag/inspirational/page/1/">inspirational</a>
+            
+            <a class="tag" href="/tag/paraphrased/page/1/">paraphrased</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it&#39;s in hot water.”</span>
+        <span>by <small class="author" itemprop="author">Eleanor Roosevelt</small>
+        <a href="/author/Eleanor-Roosevelt">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="misattributed-eleanor-roosevelt" /    > 
+            
+            <a class="tag" href="/tag/misattributed-eleanor-roosevelt/page/1/">misattributed-eleanor-roosevelt</a>
+            
+        </div>
+    </div>
+
+    <div class="quote" itemscope itemtype="http://schema.org/CreativeWork">
+        <span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>
+        <span>by <small class="author" itemprop="author">Steve Martin</small>
+        <a href="/author/Steve-Martin">(about)</a>
+        </span>
+        <div class="tags">
+            Tags:
+            <meta class="keywords" itemprop="keywords" content="humor,obvious,simile" /    > 
+            
+            <a class="tag" href="/tag/humor/page/1/">humor</a>
+            
+            <a class="tag" href="/tag/obvious/page/1/">obvious</a>
+            
+            <a class="tag" href="/tag/simile/page/1/">simile</a>
+            
+        </div>
+    </div>
+
+    <nav>
+        <ul class="pager">
+            
+            
+            <li class="next">
+                <a href="/page/2/">Next <span aria-hidden="true">&rarr;</span></a>
+            </li>
+            
+        </ul>
+    </nav>
+    </div>
+    <div class="col-md-4 tags-box">
+        
+            <h2>Top Ten tags</h2>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 28px" href="/tag/love/">love</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/inspirational/">inspirational</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 26px" href="/tag/life/">life</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 24px" href="/tag/humor/">humor</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 22px" href="/tag/books/">books</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 14px" href="/tag/reading/">reading</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 10px" href="/tag/friendship/">friendship</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/friends/">friends</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 8px" href="/tag/truth/">truth</a>
+            </span>
+            
+            <span class="tag-item">
+            <a class="tag" style="font-size: 6px" href="/tag/simile/">simile</a>
+            </span>
+            
+        
+    </div>
+</div>
+
+    </div>
+    <footer class="footer">
+        <div class="container">
+            <p class="text-muted">
+                Quotes by: <a href="https://www.goodreads.com/quotes">GoodReads.com</a>
+            </p>
+            <p class="copyright">
+                Made with <span class='sh-red'>❤</span> by <a href="https://scrapinghub.com">Scrapinghub</a>
+            </p>
+        </div>
+    </footer>
+</body>
+</html>

docs/conf.py

@@ -12,6 +12,7 @@
 # serve to show the default.
 
 import sys
+from datetime import datetime
 from os import path
 
 # If your extensions are in another directory, add it here. If the directory
@@ -27,8 +28,13 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
+    'hoverxref.extension',
+    'notfound.extension',
     'scrapydocs',
-    'sphinx.ext.autodoc'
+    'sphinx.ext.autodoc',
+    'sphinx.ext.coverage',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.viewcode',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -44,8 +50,8 @@ source_suffix = '.rst'
 master_doc = 'index'
 
 # General information about the project.
-project = u'Scrapy'
-copyright = u'2008–2018, Scrapy developers'
+project = 'Scrapy'
+copyright = '2008–{}, Scrapy developers'.format(datetime.now().year)
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -73,6 +79,8 @@ language = 'en'
 # List of documents that shouldn't be included in the build.
 #unused_docs = []
 
+exclude_patterns = ['build']
+
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
 exclude_trees = ['.build']
@@ -187,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc'
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, document class [howto/manual]).
 latex_documents = [
-  ('index', 'Scrapy.tex', u'Scrapy Documentation',
-   u'Scrapy developers', 'manual'),
+  ('index', 'Scrapy.tex', 'Scrapy Documentation',
+   'Scrapy developers', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -218,3 +226,72 @@ linkcheck_ignore = [
     'http://localhost:\d+', 'http://hg.scrapy.org',
     'http://directory.google.com/'
 ]
+
+
+# Options for the Coverage extension
+# ----------------------------------
+coverage_ignore_pyobjects = [
+    # Contract’s add_pre_hook and add_post_hook are not documented because
+    # they should be transparent to contract developers, for whom pre_hook and
+    # post_hook should be the actual concern.
+    r'\bContract\.add_(pre|post)_hook$',
+
+    # ContractsManager is an internal class, developers are not expected to
+    # interact with it directly in any way.
+    r'\bContractsManager\b$',
+
+    # For default contracts we only want to document their general purpose in
+    # their __init__ method, the methods they reimplement to achieve that purpose
+    # should be irrelevant to developers using those contracts.
+    r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
+
+    # Methods of downloader middlewares are not documented, only the classes
+    # themselves, since downloader middlewares are controlled through Scrapy
+    # settings.
+    r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
+
+    # Base classes of downloader middlewares are implementation details that
+    # are not meant for users.
+    r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
+
+    # Private exception used by the command-line interface implementation.
+    r'^scrapy\.exceptions\.UsageError',
+
+    # Methods of BaseItemExporter subclasses are only documented in
+    # BaseItemExporter.
+    r'^scrapy\.exporters\.(?!BaseItemExporter\b)\w*?\.',
+
+    # Extension behavior is only modified through settings. Methods of
+    # extension classes, as well as helper functions, are implementation
+    # details that are not documented.
+    r'^scrapy\.extensions\.[a-z]\w*?\.[A-Z]\w*?\.',  # methods
+    r'^scrapy\.extensions\.[a-z]\w*?\.[a-z]',  # helper functions
+
+    # Never documented before, and deprecated now.
+    r'^scrapy\.item\.DictItem$',
+    r'^scrapy\.linkextractors\.FilteringLinkExtractor$',
+
+    # Implementation detail of LxmlLinkExtractor
+    r'^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor',
+]
+
+
+# Options for the InterSphinx extension
+# -------------------------------------
+
+intersphinx_mapping = {
+    'coverage': ('https://coverage.readthedocs.io/en/stable', None),
+    'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
+    'pytest': ('https://docs.pytest.org/en/latest', None),
+    'python': ('https://docs.python.org/3', None),
+    'sphinx': ('https://www.sphinx-doc.org/en/master', None),
+    'tox': ('https://tox.readthedocs.io/en/latest', None),
+    'twisted': ('https://twistedmatrix.com/documents/current', None),
+    'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+}
+
+
+# Options for sphinx-hoverxref options
+# ------------------------------------
+
+hoverxref_auto_ref = True

docs/conftest.py

@@ -0,0 +1,29 @@
+import os
+from doctest import ELLIPSIS, NORMALIZE_WHITESPACE
+
+from scrapy.http.response.html import HtmlResponse
+from sybil import Sybil
+from sybil.parsers.codeblock import CodeBlockParser
+from sybil.parsers.doctest import DocTestParser
+from sybil.parsers.skip import skip
+
+
+def load_response(url, filename):
+    input_path = os.path.join(os.path.dirname(__file__), '_tests', filename)
+    with open(input_path, 'rb') as input_file:
+        return HtmlResponse(url, body=input_file.read())
+
+
+def setup(namespace):
+    namespace['load_response'] = load_response
+
+
+pytest_collect_file = Sybil(
+    parsers=[
+        DocTestParser(optionflags=ELLIPSIS | NORMALIZE_WHITESPACE),
+        CodeBlockParser(future_imports=['print_function']),
+        skip,
+    ],
+    pattern='*.rst',
+    setup=setup,
+).pytest()

docs/contributing.rst

@@ -7,7 +7,7 @@ Contributing to Scrapy
 .. important::
 
     Double check that you are reading the most recent version of this document at
-    https://doc.scrapy.org/en/master/contributing.html
+    https://docs.scrapy.org/en/master/contributing.html
 
 There are many ways to contribute to Scrapy. Here are some of them:
 
@@ -44,7 +44,7 @@ guidelines when you're going to report a new bug.
 * check the :ref:`FAQ <faq>` first to see if your issue is addressed in a
   well-known question
 
-* if you have a general question about scrapy usage, please ask it at
+* if you have a general question about Scrapy usage, please ask it at
   `Stack Overflow <https://stackoverflow.com/questions/tagged/scrapy>`__
   (use "scrapy" tag).
 
@@ -55,7 +55,7 @@ guidelines when you're going to report a new bug.
 
 * search the `scrapy-users`_ list and `Scrapy subreddit`_ to see if it has
   been discussed there, or if you're not sure if what you're seeing is a bug.
-  You can also ask in the `#scrapy` IRC channel.
+  You can also ask in the ``#scrapy`` IRC channel.
 
 * write **complete, reproducible, specific bug reports**. The smaller the test
   case, the better. Remember that other developers won't have your project to
@@ -99,6 +99,15 @@ Well-written patches should:
   the documentation changes in the same patch.  See `Documentation policies`_
   below.
 
+* if you're adding a private API, please add a regular expression to the
+  ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new
+  private API from documentation coverage checks.
+
+  To see if your private API is skipped properly, generate a documentation
+  coverage report as follows::
+
+      tox -e docs-coverage
+
 .. _submitting-patches:
 
 Submitting patches
@@ -134,7 +143,7 @@ by running ``git fetch upstream pull/$PR_NUMBER/head:$BRANCH_NAME_TO_CREATE``
 (replace 'upstream' with a remote name for scrapy repository,
 ``$PR_NUMBER`` with an ID of the pull request, and ``$BRANCH_NAME_TO_CREATE``
 with a name of the branch you want to create locally).
-See also: https://help.github.com/articles/checking-out-pull-requests-locally/#modifying-an-inactive-pull-request-locally.
+See also: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/checking-out-pull-requests-locally#modifying-an-inactive-pull-request-locally.
 
 When writing GitHub pull requests, try to keep titles short but descriptive.
 E.g. For bug #411: "Scrapy hangs if an exception raises in start_requests"
@@ -159,60 +168,80 @@ Scrapy:
 
 * Don't put your name in the code you contribute; git provides enough
   metadata to identify author of the code.
-  See https://help.github.com/articles/setting-your-username-in-git/ for
+  See https://help.github.com/en/github/using-git/setting-your-username-in-git for
   setup instructions.
 
+.. _documentation-policies:
+
 Documentation policies
 ======================
 
-* **Don't** use docstrings for documenting classes, or methods which are
-  already documented in the official (sphinx) documentation. Alternatively,
-  **do** provide a docstring, but make sure sphinx documentation uses
-  autodoc_ extension to pull the docstring. For example, the
-  :meth:`ItemLoader.add_value` method should be either
-  documented only in the sphinx documentation (not as a docstring), or
-  it should have a docstring which is pulled to sphinx documentation using
-  autodoc_ extension.
+For reference documentation of API members (classes, methods, etc.) use
+docstrings and make sure that the Sphinx documentation uses the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstrings. API reference
+documentation should follow docstring conventions (`PEP 257`_) and be
+IDE-friendly: short, to the point, and it may provide short examples.
 
-* **Do** use docstrings for documenting functions not present in the official
-  (sphinx) documentation, such as functions from ``scrapy.utils`` package and
-  its sub-modules.
+Other types of documentation, such as tutorials or topics, should be covered in
+files within the ``docs/`` directory. This includes documentation that is
+specific to an API member, but goes beyond API reference documentation.
 
-.. _autodoc: http://www.sphinx-doc.org/en/stable/ext/autodoc.html
+In any case, if something is covered in a docstring, use the
+:mod:`~sphinx.ext.autodoc` extension to pull the docstring into the
+documentation instead of duplicating the docstring in files within the
+``docs/`` directory.
 
 Tests
 =====
 
-Tests are implemented using the `Twisted unit-testing framework`_, running
-tests requires `tox`_.
+Tests are implemented using the :doc:`Twisted unit-testing framework
+<twisted:core/development/policy/test-standard>`. Running tests requires
+:doc:`tox <tox:index>`.
+
+.. _running-tests:
 
 Running tests
 -------------
 
-Make sure you have a recent enough `tox`_ installation:
+To run all tests::
 
-    ``tox --version``
-
-If your version is older than 1.7.0, please update it first:
-
-    ``pip install -U tox``
-
-To run all tests go to the root directory of Scrapy source code and run:
-
-    ``tox``
+    tox
 
 To run a specific test (say ``tests/test_loader.py``) use:
 
     ``tox -- tests/test_loader.py``
 
-To see coverage report install `coverage`_ (``pip install coverage``) and run:
+To run the tests on a specific :doc:`tox <tox:index>` environment, use
+``-e <name>`` with an environment name from ``tox.ini``. For example, to run
+the tests with Python 3.6 use::
+
+    tox -e py36
+
+You can also specify a comma-separated list of environments, and use :ref:`tox’s
+parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
+parallel::
+
+    tox -e py36,py38 -p auto
+
+To pass command-line options to :doc:`pytest <pytest:index>`, add them after
+``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
+default positional arguments defined in ``tox.ini``, so you must include those
+default positional arguments (``scrapy tests``) after ``--`` as well::
+
+    tox -- scrapy tests -x  # stop after first failure
+
+You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
+the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
+
+    tox -e py36 -- scrapy tests -n auto
+
+To see coverage report install :doc:`coverage <coverage:index>`
+(``pip install coverage``) and run:
 
     ``coverage report``
 
 see output of ``coverage --help`` for more options like html or xml report.
 
-.. _coverage: https://pypi.python.org/pypi/coverage
-
 Writing tests
 -------------
 
@@ -233,9 +262,9 @@ And their unit-tests are in::
 .. _issue tracker: https://github.com/scrapy/scrapy/issues
 .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
 .. _Scrapy subreddit: https://reddit.com/r/scrapy
-.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
-.. _pull request: https://help.github.com/send-pull-requests/
-.. _tox: https://pypi.python.org/pypi/tox
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
+.. _pull request: https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request
+.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist

docs/faq.rst

@@ -22,8 +22,8 @@ In other words, comparing `BeautifulSoup`_ (or `lxml`_) to Scrapy is like
 comparing `jinja2`_ to `Django`_.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
-.. _lxml: http://lxml.de/
-.. _jinja2: http://jinja.pocoo.org/
+.. _lxml: https://lxml.de/
+.. _jinja2: https://palletsprojects.com/p/jinja/
 .. _Django: https://www.djangoproject.com/
 
 Can I use Scrapy with BeautifulSoup?
@@ -69,11 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 2.7 and Python 3.4+
+Scrapy is supported under Python 3.5+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
-Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
 PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
+Python 2 support was dropped in Scrapy 2.0.
 
 .. note::
     For Python 3 support on Windows, it is recommended to use
@@ -129,13 +129,23 @@ Does Scrapy crawl in breadth-first or depth-first order?
 
 By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which
 basically means that it crawls in `DFO order`_. This order is more convenient
-in most cases. If you do want to crawl in true `BFO order`_, you can do it by
+in most cases.
+
+If you do want to crawl in true `BFO order`_, you can do it by
 setting the following settings::
 
     DEPTH_PRIORITY = 1
     SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
     SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
 
+While pending requests are below the configured values of
+:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
+concurrently. As a result, the first few requests of a crawl rarely follow the
+desired order. Lowering those settings to ``1`` enforces the desired order, but
+it significantly slows down the crawl as a whole.
+
+
 My Scrapy crawler has memory leaks. What can I do?
 --------------------------------------------------
 
@@ -259,7 +269,7 @@ The ``__VIEWSTATE`` parameter is used in sites built with ASP.NET/VB.NET. For
 more info on how it works see `this page`_. Also, here's an `example spider`_
 which scrapes one of these sites.
 
-.. _this page: http://search.cpan.org/~ecarroll/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
+.. _this page: https://metacpan.org/pod/release/ECARROLL/HTML-TreeBuilderX-ASP_NET-0.09/lib/HTML/TreeBuilderX/ASP_NET.pm
 .. _example spider: https://github.com/AmbientLighter/rpn-fas/blob/master/fas/spiders/rnp.py
 
 What's the best way to parse big XML/CSV data feeds?
@@ -319,6 +329,49 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
+.. _faq-split-item:
+
+How to split an item into multiple items in an item pipeline?
+-------------------------------------------------------------
+
+:ref:`Item pipelines <topics-item-pipeline>` cannot yield multiple items per
+input item. :ref:`Create a spider middleware <custom-spider-middleware>`
+instead, and use its
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+method for this purpose. For example::
+
+    from copy import deepcopy
+
+    from scrapy.item import BaseItem
+
+
+    class MultiplyItemsMiddleware:
+
+        def process_spider_output(self, response, result, spider):
+            for item in result:
+                if isinstance(item, (BaseItem, dict)):
+                    for _ in range(item['multiply_by']):
+                        yield deepcopy(item)
+
+Does Scrapy support IPv6 addresses?
+-----------------------------------
+
+Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Note that by doing so, you lose the ability to set a specific timeout for DNS requests
+(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
+
+
+.. _faq-specific-reactor:
+
+How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()`` exceptions?
+----------------------------------------------------------------------------------------------
+
+This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
+Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
+different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
+
+
+.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search

docs/index.rst

@@ -4,7 +4,13 @@
 Scrapy |version| documentation
 ==============================
 
-This documentation contains everything you need to know about Scrapy.
+Scrapy is a fast high-level `web crawling`_ and `web scraping`_ framework, used
+to crawl websites and extract structured data from their pages. It can be used
+for a wide range of purposes, from data mining to monitoring and automated
+testing.
+
+.. _web crawling: https://en.wikipedia.org/wiki/Web_crawler
+.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
 
 Getting help
 ============
@@ -152,18 +158,21 @@ Solving specific problems
    topics/practices
    topics/broad-crawls
    topics/developer-tools
+   topics/dynamic-content
    topics/leaks
    topics/media-pipeline
    topics/deploy
    topics/autothrottle
    topics/benchmarking
    topics/jobs
+   topics/coroutines
+   topics/asyncio
 
 :doc:`faq`
     Get answers to most frequently asked questions.
 
 :doc:`topics/debug`
-    Learn how to debug common problems of your scrapy spider.
+    Learn how to debug common problems of your Scrapy spider.
 
 :doc:`topics/contracts`
     Learn how to use contracts for testing your spiders.
@@ -177,6 +186,9 @@ Solving specific problems
 :doc:`topics/developer-tools`
     Learn how to scrape with your browser's developer tools.
 
+:doc:`topics/dynamic-content`
+    Read webpage data that is loaded dynamically.
+
 :doc:`topics/leaks`
     Learn how to find and get rid of memory leaks in your crawler.
 
@@ -195,6 +207,12 @@ Solving specific problems
 :doc:`topics/jobs`
     Learn how to pause and resume crawls for large spiders.
 
+:doc:`topics/coroutines`
+    Use the :ref:`coroutine syntax <async>`.
+
+:doc:`topics/asyncio`
+    Use :mod:`asyncio` and :mod:`asyncio`-powered libraries.
+
 .. _extending-scrapy:
 
 Extending Scrapy

docs/intro/install.rst

@@ -7,12 +7,12 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 2.7 and Python 3.4 or above
-under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
+Scrapy runs on Python 3.5 or above under CPython (default Python
+implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
 the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
-and OS X.
+and macOS.
 
 To install Scrapy using ``conda``, run::
 
@@ -65,7 +65,7 @@ please refer to their respective installation instructions:
 * `lxml installation`_
 * `cryptography installation`_
 
-.. _lxml installation: http://lxml.de/installation.html
+.. _lxml installation: https://lxml.de/installation.html
 .. _cryptography installation: https://cryptography.io/en/latest/installation/
 
 
@@ -78,40 +78,21 @@ TL;DR: We recommend installing Scrapy inside a virtual environment
 on all platforms.
 
 Python packages can be installed either globally (a.k.a system wide),
-or in user-space. We do not recommend installing scrapy system wide.
+or in user-space. We do not recommend installing Scrapy system wide.
 
-Instead, we recommend that you install scrapy within a so-called
-"virtual environment" (`virtualenv`_).
-Virtualenvs allow you to not conflict with already-installed Python
+Instead, we recommend that you install Scrapy within a so-called
+"virtual environment" (:mod:`venv`).
+Virtual environments allow you to not conflict with already-installed Python
 system packages (which could break some of your system tools and scripts),
 and still install packages normally with ``pip`` (without ``sudo`` and the likes).
 
-To get started with virtual environments, see `virtualenv installation instructions`_.
-To install it globally (having it globally installed actually helps here),
-it should be a matter of running::
+See :ref:`tut-venv` on how to create your virtual environment.
 
-    $ [sudo] pip install virtualenv
-
-Check this `user guide`_ on how to create your virtualenv.
-
-.. note::
-    If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
-
-Once you have created a virtualenv, you can install scrapy inside it with ``pip``,
+Once you have created a virtual environment, you can install Scrapy inside it with ``pip``,
 just like any other Python package.
 (See :ref:`platform-specific guides <intro-install-platform-notes>`
 below for non-Python dependencies that you may need to install beforehand).
 
-Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
-
-* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
-* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
-
-.. _virtualenv: https://virtualenv.pypa.io
-.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
-.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html
-.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
-
 
 .. _intro-install-platform-notes:
 
@@ -146,19 +127,15 @@ albeit with potential issues with TLS connections.
 typically too old and slow to catch up with latest Scrapy.
 
 
-To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
+To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
 these dependencies::
 
-    sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
+    sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
 
-- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
+- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
   are required for ``lxml``
 - ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
 
-If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
-
-    sudo apt-get install python3 python3-dev
-
 Inside a :ref:`virtualenv <intro-using-virtualenv>`,
 you can install Scrapy with ``pip`` after that::
 
@@ -171,11 +148,11 @@ you can install Scrapy with ``pip`` after that::
 
 .. _intro-install-macos:
 
-Mac OS X
---------
+macOS
+-----
 
 Building Scrapy's dependencies requires the presence of a C compiler and
-development headers. On OS X this is typically provided by Apple’s Xcode
+development headers. On macOS this is typically provided by Apple’s Xcode
 development tools. To install the Xcode command line tools open a terminal
 window and run::
 
@@ -211,15 +188,12 @@ solutions:
 
       brew update; brew upgrade python
 
-* *(Optional)* Install Scrapy inside an isolated python environment.
+*   *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
+    <intro-using-virtualenv>`.
 
-  This method is a workaround for the above OS X issue, but it's an overall
+  This method is a workaround for the above macOS issue, but it's an overall
   good practice for managing dependencies and can complement the first method.
 
-  `virtualenv`_ is a tool you can use to create virtual environments in python.
-  We recommended reading a tutorial like
-  http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started.
-
 After any of these workarounds you should be able to install Scrapy::
 
   pip install Scrapy
@@ -231,17 +205,17 @@ PyPy
 We recommend using the latest PyPy version. The version tested is 5.9.0.
 For PyPy3, only Linux installation was tested.
 
-Most scrapy dependencides now have binary wheels for CPython, but not for PyPy.
+Most Scrapy dependencides now have binary wheels for CPython, but not for PyPy.
 This means that these dependecies will be built during installation.
-On OS X, you are likely to face an issue with building Cryptography dependency,
+On macOS, you are likely to face an issue with building Cryptography dependency,
 solution to this problem is described
 `here <https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,
 that is to ``brew install openssl`` and then export the flags that this command
-recommends (only needed when installing scrapy). Installing on Linux has no special
+recommends (only needed when installing Scrapy). Installing on Linux has no special
 issues besides installing build dependencies.
-Installing scrapy with PyPy on Windows is not tested.
+Installing Scrapy with PyPy on Windows is not tested.
 
-You can check that scrapy is installed correctly by running ``scrapy bench``.
+You can check that Scrapy is installed correctly by running ``scrapy bench``.
 If this command gives errors such as
 ``TypeError: ... got 2 unexpected keyword arguments``, this means
 that setuptools was unable to pick up one PyPy-specific dependency.
@@ -278,17 +252,17 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
 
 .. _Python: https://www.python.org/
 .. _pip: https://pip.pypa.io/en/latest/installing/
-.. _lxml: http://lxml.de/
-.. _parsel: https://pypi.python.org/pypi/parsel
-.. _w3lib: https://pypi.python.org/pypi/w3lib
-.. _twisted: https://twistedmatrix.com/
-.. _cryptography: https://cryptography.io/
-.. _pyOpenSSL: https://pypi.python.org/pypi/pyOpenSSL
+.. _lxml: https://lxml.de/index.html
+.. _parsel: https://pypi.org/project/parsel/
+.. _w3lib: https://pypi.org/project/w3lib/
+.. _twisted: https://twistedmatrix.com/trac/
+.. _cryptography: https://cryptography.io/en/latest/
+.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
 .. _setuptools: https://pypi.python.org/pypi/setuptools
 .. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
 .. _homebrew: https://brew.sh/
 .. _zsh: https://www.zsh.org/
 .. _Scrapinghub: https://scrapinghub.com
 .. _Anaconda: https://docs.anaconda.com/anaconda/
-.. _Miniconda: https://conda.io/docs/user-guide/install/index.html
+.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
 .. _conda-forge: https://conda-forge.org/

docs/intro/overview.rst

@@ -26,7 +26,7 @@ http://quotes.toscrape.com, following the pagination::
 
 
     class QuotesSpider(scrapy.Spider):
-        name = "quotes"
+        name = 'quotes'
         start_urls = [
             'http://quotes.toscrape.com/tag/humor/',
         ]
@@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').get(),
                     'author': quote.xpath('span/small/text()').get(),
+                    'text': quote.css('span.text::text').get(),
                 }
 
             next_page = response.css('li.next a::attr("href")').get()

docs/intro/tutorial.rst

@@ -78,9 +78,9 @@ Our first Spider
 
 Spiders are classes that you define and that Scrapy uses to scrape information
 from a website (or a group of websites). They must subclass
-:class:`scrapy.Spider` and define the initial requests to make, optionally how
-to follow links in the pages, and how to parse the downloaded page content to
-extract data.
+:class:`~scrapy.spiders.Spider` and define the initial requests to make,
+optionally how to follow links in the pages, and how to parse the downloaded
+page content to extract data.
 
 This is the code for our first Spider. Save it in a file named
 ``quotes_spider.py`` under the ``tutorial/spiders`` directory in your project::
@@ -205,14 +205,14 @@ Extracting data
 ---------------
 
 The best way to learn how to extract data with Scrapy is trying selectors
-using the shell :ref:`Scrapy shell <topics-shell>`. Run::
+using the :ref:`Scrapy shell <topics-shell>`. Run::
 
     scrapy shell 'http://quotes.toscrape.com/page/1/'
 
 .. note::
 
    Remember to always enclose urls in quotes when running Scrapy shell from
-   command-line, otherwise urls containing arguments (ie. ``&`` character)
+   command-line, otherwise urls containing arguments (i.e. ``&`` character)
    will not work.
 
    On Windows, use double quotes instead::
@@ -235,13 +235,16 @@ You will see something like::
     [s]   shelp()           Shell help (print this help)
     [s]   fetch(req_or_url) Fetch request (or URL) and update local objects
     [s]   view(response)    View response in a browser
-    >>>
 
 Using the shell, you can try selecting elements using `CSS`_ with the response
-object::
+object:
 
-    >>> response.css('title')
-    [<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
+.. invisible-code-block: python
+
+    response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
+
+>>> response.css('title')
+[<Selector xpath='descendant-or-self::title' data='<title>Quotes to Scrape</title>'>]
 
 The result of running ``response.css('title')`` is a list-like object called
 :class:`~scrapy.selector.SelectorList`, which represents a list of
@@ -249,30 +252,30 @@ The result of running ``response.css('title')`` is a list-like object called
 and allow you to run further queries to fine-grain the selection or extract the
 data.
 
-To extract the text from the title above, you can do::
+To extract the text from the title above, you can do:
 
-    >>> response.css('title::text').getall()
-    ['Quotes to Scrape']
+>>> response.css('title::text').getall()
+['Quotes to Scrape']
 
 There are two things to note here: one is that we've added ``::text`` to the
 CSS query, to mean we want to select only the text elements directly inside
 ``<title>`` element.  If we don't specify ``::text``, we'd get the full title
-element, including its tags::
+element, including its tags:
 
-    >>> response.css('title').getall()
-    ['<title>Quotes to Scrape</title>']
+>>> response.css('title').getall()
+['<title>Quotes to Scrape</title>']
 
 The other thing is that the result of calling ``.getall()`` is a list: it is
 possible that a selector returns more than one result, so we extract them all.
-When you know you just want the first result, as in this case, you can do::
+When you know you just want the first result, as in this case, you can do:
 
-    >>> response.css('title::text').get()
-    'Quotes to Scrape'
+>>> response.css('title::text').get()
+'Quotes to Scrape'
 
-As an alternative, you could've written::
+As an alternative, you could've written:
 
-    >>> response.css('title::text')[0].get()
-    'Quotes to Scrape'
+>>> response.css('title::text')[0].get()
+'Quotes to Scrape'
 
 However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
 instance avoids an ``IndexError`` and returns ``None`` when it doesn't
@@ -285,36 +288,36 @@ to be scraped, you can at least get **some** data.
 Besides the :meth:`~scrapy.selector.SelectorList.getall` and
 :meth:`~scrapy.selector.SelectorList.get` methods, you can also use
 the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
-expressions`_::
+expressions`_:
 
-    >>> response.css('title::text').re(r'Quotes.*')
-    ['Quotes to Scrape']
-    >>> response.css('title::text').re(r'Q\w+')
-    ['Quotes']
-    >>> response.css('title::text').re(r'(\w+) to (\w+)')
-    ['Quotes', 'Scrape']
+>>> response.css('title::text').re(r'Quotes.*')
+['Quotes to Scrape']
+>>> response.css('title::text').re(r'Q\w+')
+['Quotes']
+>>> response.css('title::text').re(r'(\w+) to (\w+)')
+['Quotes', 'Scrape']
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools to inspect the HTML and come up
-with a selector (see section about :ref:`topics-developer-tools`).
+You can use your browser's developer tools to inspect the HTML and come up
+with a selector (see :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.
 
 .. _regular expressions: https://docs.python.org/3/library/re.html
-.. _Selector Gadget: http://selectorgadget.com/
+.. _Selector Gadget: https://selectorgadget.com/
 
 
 XPath: a brief intro
 ^^^^^^^^^^^^^^^^^^^^
 
-Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
+Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
 
-    >>> response.xpath('//title')
-    [<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
-    >>> response.xpath('//title/text()').get()
-    'Quotes to Scrape'
+>>> response.xpath('//title')
+[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
+>>> response.xpath('//title/text()').get()
+'Quotes to Scrape'
 
 XPath expressions are very powerful, and are the foundation of Scrapy
 Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
@@ -334,7 +337,7 @@ recommend `this tutorial to learn XPath through examples
 <http://zvon.org/comp/r/tut-XPath_1.html>`_, and `this tutorial to learn "how
 to think in XPath" <http://plasmasturm.org/log/xpath101/>`_.
 
-.. _XPath: https://www.w3.org/TR/xpath
+.. _XPath: https://www.w3.org/TR/xpath/all/
 .. _CSS: https://www.w3.org/TR/selectors
 
 Extracting quotes and authors
@@ -369,45 +372,53 @@ we want::
 
     $ scrapy shell 'http://quotes.toscrape.com'
 
-We get a list of selectors for the quote HTML elements with::
+We get a list of selectors for the quote HTML elements with:
 
-    >>> response.css("div.quote")
+>>> response.css("div.quote")
+[<Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+ <Selector xpath="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+ ...]
 
 Each of the selectors returned by the query above allows us to run further
 queries over their sub-elements. Let's assign the first selector to a
-variable, so that we can run our CSS selectors directly on a particular quote::
+variable, so that we can run our CSS selectors directly on a particular quote:
 
-    >>> quote = response.css("div.quote")[0]
+>>> quote = response.css("div.quote")[0]
 
-Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
-using the ``quote`` object we just created::
+Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
+using the ``quote`` object we just created:
 
-    >>> title = quote.css("span.text::text").get()
-    >>> title
-    '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
-    >>> author = quote.css("small.author::text").get()
-    >>> author
-    'Albert Einstein'
+>>> text = quote.css("span.text::text").get()
+>>> text
+'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
+>>> author = quote.css("small.author::text").get()
+>>> author
+'Albert Einstein'
 
 Given that the tags are a list of strings, we can use the ``.getall()`` method
-to get all of them::
+to get all of them:
 
-    >>> tags = quote.css("div.tags a.tag::text").getall()
-    >>> tags
-    ['change', 'deep-thoughts', 'thinking', 'world']
+>>> tags = quote.css("div.tags a.tag::text").getall()
+>>> tags
+['change', 'deep-thoughts', 'thinking', 'world']
+
+.. invisible-code-block: python
+
+  from sys import version_info
+
+.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
 
 Having figured out how to extract each bit, we can now iterate over all the
-quotes elements and put them together into a Python dictionary::
+quotes elements and put them together into a Python dictionary:
 
-    >>> for quote in response.css("div.quote"):
-    ...     text = quote.css("span.text::text").get()
-    ...     author = quote.css("small.author::text").get()
-    ...     tags = quote.css("div.tags a.tag::text").getall()
-    ...     print(dict(text=text, author=author, tags=tags))
-    {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
-    {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
-        ... a few more of these, omitted for brevity
-    >>>
+>>> for quote in response.css("div.quote"):
+...     text = quote.css("span.text::text").get()
+...     author = quote.css("small.author::text").get()
+...     tags = quote.css("div.tags a.tag::text").getall()
+...     print(dict(text=text, author=author, tags=tags))
+{'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
+{'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
+...
 
 Extracting data in our spider
 -----------------------------
@@ -505,23 +516,23 @@ markup:
         </li>
     </ul>
 
-We can try extracting it in the shell::
+We can try extracting it in the shell:
 
-    >>> response.css('li.next a').get()
-    '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
+>>> response.css('li.next a').get()
+'<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
-Scrapy supports a CSS extension that let's you select the attribute contents,
-like this::
+Scrapy supports a CSS extension that lets you select the attribute contents,
+like this:
 
-    >>> response.css('li.next a::attr(href)').get()
-    '/page/2/'
+>>> response.css('li.next a::attr(href)').get()
+'/page/2/'
 
 There is also an ``attrib`` property available
-(see :ref:`selecting-attributes` for more)::
+(see :ref:`selecting-attributes` for more):
 
-    >>> response.css('li.next a').attrib['href']
-    '/page/2'
+>>> response.css('li.next a').attrib['href']
+'/page/2/'
 
 Let's see now our spider modified to recursively follow the link to the next
 page, extracting data from it::
@@ -605,21 +616,25 @@ instance; you still have to yield this Request.
 You can also pass a selector to ``response.follow`` instead of a string;
 this selector should extract necessary attributes::
 
-    for href in response.css('li.next a::attr(href)'):
+    for href in response.css('ul.pager a::attr(href)'):
         yield response.follow(href, callback=self.parse)
 
 For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
 attribute automatically. So the code can be shortened further::
 
-    for a in response.css('li.next a'):
+    for a in response.css('ul.pager a'):
         yield response.follow(a, callback=self.parse)
 
-.. note::
+To create multiple requests from an iterable, you can use
+:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` instead::
+
+    anchors = response.css('ul.pager a')
+    yield from response.follow_all(anchors, callback=self.parse)
+
+or, shortening it further::
+
+    yield from response.follow_all(css='ul.pager a', callback=self.parse)
 
-    ``response.follow(response.css('li.next a'))`` is not valid because
-    ``response.css`` returns a list-like object with selectors for all results,
-    not a single selector. A ``for`` loop like in the example above, or
-    ``response.follow(response.css('li.next a')[0])`` is fine.
 
 More examples and patterns
 --------------------------
@@ -636,13 +651,11 @@ this time for scraping author information::
         start_urls = ['http://quotes.toscrape.com/']
 
         def parse(self, response):
-            # follow links to author pages
-            for href in response.css('.author + a::attr(href)'):
-                yield response.follow(href, self.parse_author)
+            author_page_links = response.css('.author + a')
+            yield from response.follow_all(author_page_links, self.parse_author)
 
-            # follow pagination links
-            for href in response.css('li.next a::attr(href)'):
-                yield response.follow(href, self.parse)
+            pagination_links = response.css('li.next a')
+            yield from response.follow_all(pagination_links, self.parse)
 
         def parse_author(self, response):
             def extract_with_css(query):
@@ -658,8 +671,10 @@ This spider will start from the main page, it will follow all the links to the
 authors pages calling the ``parse_author`` callback for each of them, and also
 the pagination links with the ``parse`` callback as we saw before.
 
-Here we're passing callbacks to ``response.follow`` as positional arguments
-to make the code shorter; it also works for ``scrapy.Request``.
+Here we're passing callbacks to
+:meth:`response.follow_all <scrapy.http.TextResponse.follow_all>` as positional
+arguments to make the code shorter; it also works for
+:class:`~scrapy.http.Request`.
 
 The ``parse_author`` callback defines a helper function to extract and cleanup the
 data from a CSS query and yields the Python dict with the author data.

docs/requirements.txt

@@ -1,2 +1,4 @@
-Sphinx>=1.6
-sphinx_rtd_theme
+Sphinx>=2.1
+sphinx-hoverxref
+sphinx-notfound-page
+sphinx_rtd_theme

docs/topics/api.rst

@@ -94,11 +94,13 @@ how you :ref:`configure the downloader middlewares
     .. method:: crawl(\*args, \**kwargs)
 
         Starts the crawler by instantiating its spider class with the given
-        `args` and `kwargs` arguments, while setting the execution engine in
+        ``args`` and ``kwargs`` arguments, while setting the execution engine in
         motion.
 
         Returns a deferred that is fired when the crawl is finished.
 
+    .. automethod:: stop
+
 .. autoclass:: CrawlerRunner
    :members:
 
@@ -154,7 +156,7 @@ Settings API
 SpiderLoader API
 ================
 
-.. module:: scrapy.loader
+.. module:: scrapy.spiderloader
    :synopsis: The spider loader
 
 .. class:: SpiderLoader
@@ -180,7 +182,7 @@ SpiderLoader API
     .. method:: load(spider_name)
 
        Get the Spider class with the given name. It'll look into the previously
-       loaded spiders for a spider class with name `spider_name` and will raise
+       loaded spiders for a spider class with name ``spider_name`` and will raise
        a KeyError if not found.
 
        :param spider_name: spider class name
@@ -271,5 +273,3 @@ class (which they all inherit from).
 
         Close the given spider. After this is called, no more specific stats
         can be accessed or collected.
-
-.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html

docs/topics/architecture.rst

@@ -166,11 +166,10 @@ for concurrency.
 For more information about asynchronous programming and Twisted see these
 links:
 
-* `Introduction to Deferreds in Twisted`_
+* :doc:`twisted:core/howto/defer-intro`
 * `Twisted - hello, asynchronous programming`_
 * `Twisted Introduction - Krondo`_
 
 .. _Twisted: https://twistedmatrix.com/trac/
-.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
-.. _Twisted - hello, asynchronous programming: http://jessenoller.com/2009/02/11/twisted-hello-asynchronous-programming/
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
 .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/

docs/topics/asyncio.rst

@@ -0,0 +1,28 @@
+=======
+asyncio
+=======
+
+.. versionadded:: 2.0
+
+Scrapy has partial support :mod:`asyncio`. After you :ref:`install the asyncio
+reactor <install-asyncio>`, you may use :mod:`asyncio` and
+:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
+
+.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
+             versions may introduce related changes without a deprecation
+             period or warning.
+
+.. _install-asyncio:
+
+Installing the asyncio reactor
+==============================
+
+To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to
+``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``.
+
+If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to
+install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
+reactor manually. You can do that using
+:func:`~scrapy.utils.reactor.install_reactor`::
+
+    install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')

docs/topics/autothrottle.rst

@@ -11,7 +11,7 @@ Design goals
 ============
 
 1. be nicer to sites instead of using default download delay of zero
-2. automatically adjust scrapy to the optimum crawling speed, so the user
+2. automatically adjust Scrapy to the optimum crawling speed, so the user
    doesn't have to tune the download delays to find the optimum one.
    The user only needs to specify the maximum concurrent requests
    it allows, and the extension does the rest.

docs/topics/broad-crawls.rst

@@ -39,24 +39,50 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
 concrete suggestions of Scrapy settings to tune in order to achieve an
 efficient broad crawl.
 
+.. _broad-crawls-scheduler-priority-queue:
+
+Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
+=================================================
+
+Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
+It works best during single-domain crawl. It does not work well with crawling
+many different domains in parallel
+
+To apply the recommended priority queue use::
+
+    SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+
+.. _broad-crawls-concurrency:
+
 Increase concurrency
 ====================
 
 Concurrency is the number of requests that are processed in parallel. There is
-a global limit and a per-domain limit.
+a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
+can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per
+IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
+
+.. note:: The scheduler priority queue :ref:`recommended for broad crawls
+          <broad-crawls-scheduler-priority-queue>` does not support
+          :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 The default global concurrency limit in Scrapy is not suitable for crawling
 many different domains in parallel, so you will want to increase it. How much
-to increase it will depend on how much CPU you crawler will have available. A
-good starting point is ``100``, but the best way to find out is by doing some
-trials and identifying at what concurrency your Scrapy process gets CPU
-bounded. For optimum performance, you should pick a concurrency where CPU usage
-is at 80-90%.
+to increase it will depend on how much CPU and memory you crawler will have
+available.
 
-To increase the global concurrency use::
+A good starting point is ``100``::
 
     CONCURRENT_REQUESTS = 100
 
+But the best way to find out is by doing some trials and identifying at what
+concurrency your Scrapy process gets CPU bounded. For optimum performance, you
+should pick a concurrency where CPU usage is at 80-90%.
+
+Increasing concurrency also increases memory usage. If memory usage is a
+concern, you might need to lower your global concurrency limit accordingly.
+
+
 Increase Twisted IO thread pool maximum size
 ============================================
 
@@ -85,7 +111,7 @@ When doing broad crawls you are often only interested in the crawl rates you
 get and any errors found. These stats are reported by Scrapy when using the
 ``INFO`` log level. In order to save CPU (and log storage requirements) you
 should not use ``DEBUG`` log level when preforming large broad crawls in
-production. Using ``DEBUG`` level when developing your (broad) crawler may be 
+production. Using ``DEBUG`` level when developing your (broad) crawler may be
 fine though.
 
 To set the log level use::
@@ -162,4 +188,33 @@ AjaxCrawlMiddleware helps to crawl them correctly.
 It is turned OFF by default because it has some performance overhead,
 and enabling it for focused crawls doesn't make much sense.
 
-.. _ajax crawlable: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+.. _ajax crawlable: https://developers.google.com/search/docs/ajax-crawling/docs/getting-started
+
+.. _broad-crawls-bfo:
+
+Crawl in BFO order
+==================
+
+:ref:`Scrapy crawls in DFO order by default <faq-bfo-dfo>`.
+
+In broad crawls, however, page crawling tends to be faster than page
+processing. As a result, unprocessed early requests stay in memory until the
+final depth is reached, which can significantly increase memory usage.
+
+:ref:`Crawl in BFO order <faq-bfo-dfo>` instead to save memory.
+
+
+Be mindful of memory leaks
+==========================
+
+If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
+BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
+<broad-crawls-concurrency>` you should :ref:`debug your memory leaks
+<topics-leaks>`.
+
+
+Install a specific Twisted reactor
+==================================
+
+If the crawl is exceeding the system's capabilities, you might want to try
+installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.

docs/topics/commands.rst

@@ -1,3 +1,5 @@
+.. highlight:: none
+
 .. _topics-commands:
 
 =================
@@ -27,7 +29,7 @@ in standard locations:
 1. ``/etc/scrapy.cfg`` or ``c:\scrapy\scrapy.cfg`` (system-wide),
 2. ``~/.config/scrapy.cfg`` (``$XDG_CONFIG_HOME``) and ``~/.scrapy.cfg`` (``$HOME``)
    for global (user-wide) settings, and
-3. ``scrapy.cfg`` inside a scrapy project's root (see next section).
+3. ``scrapy.cfg`` inside a Scrapy project's root (see next section).
 
 Settings from these files are merged in the listed order of preference:
 user-defined values have higher priority than system-wide defaults
@@ -66,7 +68,9 @@ structure by default, similar to this::
 
 The directory where the ``scrapy.cfg`` file resides is known as the *project
 root directory*. That file contains the name of the python module that defines
-the project settings. Here is an example::
+the project settings. Here is an example:
+
+.. code-block:: ini
 
     [settings]
     default = myproject.settings
@@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be
 shared by multiple Scrapy projects, each with its own settings module.
 
 In that case, you must define one or more aliases for those settings modules
-under ``[settings]`` in your ``scrapy.cfg`` file::
+under ``[settings]`` in your ``scrapy.cfg`` file:
+
+.. code-block:: ini
 
     [settings]
     default = myproject1.settings
@@ -277,6 +283,8 @@ check
 
 Run contract checks.
 
+.. skip: start
+
 Usage examples::
 
     $ scrapy check -l
@@ -294,6 +302,8 @@ Usage examples::
     [FAILED] first_spider:parse
     >>> Returned 92 requests, expected 0..4
 
+.. skip: end
+
 .. command:: list
 
 list
@@ -461,6 +471,9 @@ Supported options:
 * ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
+* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
+  This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}'
+
 * ``--pipelines``: process items through pipelines
 
 * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`
@@ -478,6 +491,8 @@ Supported options:
 
 * ``--verbose`` or ``-v``: display information for each depth level
 
+.. skip: start
+
 Usage example::
 
     $ scrapy parse http://www.example.com/ -c parse_item
@@ -492,6 +507,8 @@ Usage example::
     # Requests  -----------------------------------------------------------------
     []
 
+.. skip: end
+
 
 .. command:: settings
 
@@ -570,7 +587,9 @@ Default: ``''`` (empty string)
 A module to use for looking up custom Scrapy commands. This is used to add custom
 commands for your Scrapy project.
 
-Example::
+Example:
+
+.. code-block:: python
 
     COMMANDS_MODULE = 'mybot.commands'
 
@@ -585,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
 ``scrapy.commands`` section in the entry points of the library ``setup.py``
 file.
 
-The following example adds ``my_command`` command::
+The following example adds ``my_command`` command:
+
+.. skip: next
+
+.. code-block:: python
 
   from setuptools import setup, find_packages
 

docs/topics/contracts.rst

@@ -6,10 +6,6 @@ Spiders Contracts
 
 .. versionadded:: 0.15
 
-.. note:: This is a new feature (introduced in Scrapy 0.15) and may be subject
-   to minor functionality/API updates. Check the :ref:`release notes <news>` to
-   be notified of updates.
-
 Testing spiders can get particularly annoying and while nothing prevents you
 from writing unit tests the task gets cumbersome quickly. Scrapy offers an
 integrated way of testing your spiders by the means of contracts.
@@ -35,12 +31,20 @@ This callback is tested using three built-in contracts:
 
 .. class:: UrlContract
 
-    This contract (``@url``) sets the sample url used when checking other
+    This contract (``@url``) sets the sample URL used when checking other
     contract conditions for this spider. This contract is mandatory. All
     callbacks lacking this contract are ignored when running the checks::
 
     @url url
 
+.. class:: CallbackKeywordArgumentsContract
+
+    This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.http.Request.cb_kwargs>`
+    attribute for the sample request. It must be a valid JSON dictionary.
+    ::
+
+    @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
+
 .. class:: ReturnsContract
 
     This contract (``@returns``) sets lower and upper bounds for the items and
@@ -60,7 +64,7 @@ Use the :command:`check` command to run the contract checks.
 Custom Contracts
 ================
 
-If you find you need more power than the built-in scrapy contracts you can
+If you find you need more power than the built-in Scrapy contracts you can
 create and load your own contracts in the project by using the
 :setting:`SPIDER_CONTRACTS` setting::
 
@@ -69,7 +73,7 @@ create and load your own contracts in the project by using the
         'myproject.contracts.ItemValidate': 10,
     }
 
-Each contract must inherit from :class:`scrapy.contracts.Contract` and can
+Each contract must inherit from :class:`~scrapy.contracts.Contract` and can
 override three methods:
 
 .. module:: scrapy.contracts
@@ -102,9 +106,14 @@ override three methods:
         This allows processing the output of the callback. Iterators are
         converted listified before being passed to this hook.
 
+Raise :class:`~scrapy.exceptions.ContractFail` from
+:class:`~scrapy.contracts.Contract.pre_process` or
+:class:`~scrapy.contracts.Contract.post_process` if expectations are not met:
+
+.. autoclass:: scrapy.exceptions.ContractFail
+
 Here is a demo contract which checks the presence of a custom header in the
-response received. Raise :class:`scrapy.exceptions.ContractFail` in order to
-get the failures pretty printed::
+response received::
 
     from scrapy.contracts import Contract
     from scrapy.exceptions import ContractFail
@@ -120,3 +129,24 @@ get the failures pretty printed::
             for header in self.args:
                 if header not in response.headers:
                     raise ContractFail('X-CustomHeader not present')
+
+.. _detecting-contract-check-runs:
+
+Detecting check runs
+====================
+
+When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
+set to the ``true`` string. You can use `os.environ`_ to perform any change to
+your spiders or your settings when ``scrapy check`` is used::
+
+    import os
+    import scrapy
+
+    class ExampleSpider(scrapy.Spider):
+        name = 'example'
+
+        def __init__(self):
+            if os.environ.get('SCRAPY_CHECK'):
+                pass  # Do some scraper adjustments when a check is running
+
+.. _os.environ: https://docs.python.org/3/library/os.html#os.environ

docs/topics/coroutines.rst

@@ -0,0 +1,110 @@
+==========
+Coroutines
+==========
+
+.. versionadded:: 2.0
+
+Scrapy has :ref:`partial support <coroutine-support>` for the
+:ref:`coroutine syntax <async>`.
+
+.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
+             versions may introduce related API and behavior changes without a
+             deprecation period or warning.
+
+.. _coroutine-support:
+
+Supported callables
+===================
+
+The following callables may be defined as coroutines using ``async def``, and
+hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
+
+-   :class:`~scrapy.http.Request` callbacks.
+
+    The following are known caveats of the current implementation that we aim
+    to address in future versions of Scrapy:
+
+    -   The callback output is not processed until the whole callback finishes.
+
+        As a side effect, if the callback raises an exception, none of its
+        output is processed.
+
+    -   Because `asynchronous generators were introduced in Python 3.6`_, you
+        can only use ``yield`` if you are using Python 3.6 or later.
+
+        If you need to output multiple items or requests and you are using
+        Python 3.5, return an iterable (e.g. a list) instead.
+
+-   The :meth:`process_item` method of
+    :ref:`item pipelines <topics-item-pipeline>`.
+
+-   The
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`,
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`,
+    and
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
+    methods of
+    :ref:`downloader middlewares <topics-downloader-middleware-custom>`.
+
+-   :ref:`Signal handlers that support deferreds <signal-deferred>`.
+
+.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
+
+Usage
+=====
+
+There are several use cases for coroutines in Scrapy. Code that would
+return Deferreds when written for previous Scrapy versions, such as downloader
+middlewares and signal handlers, can be rewritten to be shorter and cleaner::
+
+    class DbPipeline:
+        def _update_item(self, data, item):
+            item['field'] = data
+            return item
+
+        def process_item(self, item, spider):
+            dfd = db.get_some_data(item['id'])
+            dfd.addCallback(self._update_item, item)
+            return dfd
+
+becomes::
+
+    class DbPipeline:
+        async def process_item(self, item, spider):
+            item['field'] = await db.get_some_data(item['id'])
+            return item
+
+Coroutines may be used to call asynchronous code. This includes other
+coroutines, functions that return Deferreds and functions that return
+`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use
+many useful Python libraries providing such code::
+
+    class MySpider(Spider):
+        # ...
+        async def parse_with_deferred(self, response):
+            additional_response = await treq.get('https://additional.url')
+            additional_data = await treq.content(additional_response)
+            # ... use response and additional_data to yield items and requests
+
+        async def parse_with_asyncio(self, response):
+            async with aiohttp.ClientSession() as session:
+                async with session.get('https://additional.url') as additional_response:
+                    additional_data = await r.text()
+            # ... use response and additional_data to yield items and requests
+
+.. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
+          :mod:`asyncio` loop and to use them you need to
+          :doc:`enable asyncio support in Scrapy<asyncio>`.
+
+Common use cases for asynchronous code include:
+
+* requesting data from websites, databases and other services (in callbacks,
+  pipelines and middlewares);
+* storing data in databases (in pipelines and middlewares);
+* delaying the spider initialization until some external event (in the
+  :signal:`spider_opened` handler);
+* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
+  :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
+
+.. _aio-libs: https://github.com/aio-libs
+.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable

docs/topics/debug.rst

@@ -5,7 +5,7 @@ Debugging Spiders
 =================
 
 This document explains the most common techniques for debugging spiders.
-Consider the following scrapy spider below::
+Consider the following Scrapy spider below::
 
     import scrapy
     from myproject.items import MyItem
@@ -28,16 +28,15 @@ Consider the following scrapy spider below::
             item = MyItem()
             # populate `item` fields
             # and extract item_details_url
-            yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
+            yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
 
-        def parse_details(self, response):
-            item = response.meta['item']
+        def parse_details(self, response, item):
             # populate more `item` fields
             return item
 
 Basically this is a simple spider which parses two pages of items (the
 start_urls). Items also have a details page with additional information, so we
-use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a
+use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
 partially populated item.
 
 
@@ -49,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
 of the spider at the method level. It has the advantage of being flexible and
 simple to use, but does not allow debugging code inside a method.
 
+.. highlight:: none
+
+.. skip: start
+
 In order to see the item scraped from a specific url::
 
     $ scrapy parse --spider=myspider -c parse_item -d 2 <item_url>
@@ -86,6 +89,8 @@ using::
 
     $ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
 
+.. skip: end
+
 
 Scrapy Shell
 ============
@@ -95,13 +100,14 @@ spider, it is of little help to check what happens inside a callback, besides
 showing the response received and the output. How to debug the situation when
 ``parse_details`` sometimes receives no item?
 
+.. highlight:: python
+
 Fortunately, the :command:`shell` is your bread and butter in this case (see
 :ref:`topics-shell-inspect-response`)::
 
     from scrapy.shell import inspect_response
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item
@@ -134,8 +140,7 @@ Logging is another useful option for getting information about your spider run.
 Although not as convenient, it comes with the advantage that the logs will be
 available in all future runs should they be necessary again::
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item

docs/topics/developer-tools.rst

@@ -39,7 +39,7 @@ Therefore, you should keep in mind the following things:
 .. _topics-inspector:
 
 Inspecting a website
-===================================
+====================
 
 By far the most handy feature of the Developer Tools is the `Inspector` 
 feature, which allows you to inspect the underlying HTML code of 
@@ -79,13 +79,23 @@ sections and tags of a webpage, which greatly improves readability. You can
 expand and collapse a tag by clicking on the arrow in front of it or by double
 clicking directly on the tag. If we expand the ``span`` tag with the ``class=
 "text"`` we will see the quote-text we clicked on. The `Inspector` lets you
-copy XPaths to selected elements. Let's try it out: Right-click on the ``span`` 
-tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
+copy XPaths to selected elements. Let's try it out.
+
+First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
+
+.. code-block:: none
 
     $ scrapy shell "http://quotes.toscrape.com/"
-    (...)
-    >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
-    ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
+
+Then, back to your web browser, right-click on the ``span`` tag, select
+``Copy > XPath`` and paste it in the Scrapy shell like so:
+
+.. invisible-code-block: python
+
+    response = load_response('http://quotes.toscrape.com/', 'quotes.html')
+
+>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
 
 Adding ``text()`` at the end we are able to extract the first quote with this 
 basic selector. But this XPath is not really that clever. All it does is
@@ -112,13 +122,13 @@ see each quote:
 
 With this knowledge we can refine our XPath: Instead of a path to follow,
 we'll simply select all ``span`` tags with the ``class="text"`` by using 
-the `has-class-extension`_:: 
+the `has-class-extension`_:
 
-    >>> response.xpath('//span[has-class("text")]/text()').getall()
-   ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”,
-    '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
-    '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
-    (...)]
+>>> response.xpath('//span[has-class("text")]/text()').getall()
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
+'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
+'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
+...]
 
 And with one simple, cleverer XPath we are able to extract all quotes from 
 the page. We could have constructed a loop over our first XPath to increase 
@@ -132,7 +142,7 @@ a use case:
 
 Say you want to find the ``Next`` button on the page. Type ``Next`` into the 
 search bar on the top right of the `Inspector`. You should get two results. 
-The first is a ``li`` tag with the ``class="text"``, the second the text 
+The first is a ``li`` tag with the ``class="next"``, the second the text 
 of an ``a`` tag. Right click on the ``a`` tag and select ``Scroll into View``.
 If you hover over the tag, you'll see the button highlighted. From here
 we could easily create a :ref:`Link Extractor <topics-link-extractors>` to 
@@ -159,7 +169,11 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page,
 but instead of the above-mentioned ``Next`` button, the page 
 automatically loads new quotes when you scroll to the bottom. We 
 could go ahead and try out different XPaths directly, but instead 
-we'll check another quite useful command from the scrapy shell::
+we'll check another quite useful command from the Scrapy shell:
+
+.. skip: next
+
+.. code-block:: none
 
   $ scrapy shell "quotes.toscrape.com/scroll"
   (...)
@@ -203,7 +217,7 @@ where our quotes are coming from:
 First click on the request with the name ``scroll``. On the right 
 you can now inspect the request. In ``Headers`` you'll find details
 about the request headers, such as the URL, the method, the IP-address,
-and so on. We'll ignore the other tabs and click directly on ``Reponse``.
+and so on. We'll ignore the other tabs and click directly on ``Response``.
 
 What you should see in the ``Preview`` pane is the rendered HTML-code, 
 that is exactly what we saw when we called ``view(response)`` in the 
@@ -233,7 +247,7 @@ also request each page to get every quote on the site::
         name = 'quote'
         allowed_domains = ['quotes.toscrape.com']
         page = 1
-        start_urls = ['http://quotes.toscrape.com/api/quotes?page=1]
+        start_urls = ['http://quotes.toscrape.com/api/quotes?page=1']
 
         def parse(self, response):
             data = json.loads(response.text)
@@ -252,9 +266,33 @@ If the handy ``has_next`` element is ``true`` (try loading
 `quotes.toscrape.com/api/quotes?page=10`_ in your browser or a
 page-number greater than 10), we increment the ``page`` attribute 
 and ``yield`` a new request, inserting the incremented page-number 
-into our ``url``. 
+into our ``url``.
 
-You can see that with a few inspections in the `Network`-tool we 
+.. _requests-from-curl:
+
+In more complex websites, it could be difficult to easily reproduce the
+requests, as we could need to add ``headers`` or ``cookies`` to make it work.
+In those cases you can export the requests in `cURL <https://curl.haxx.se/>`_
+format, by right-clicking on each of them in the network tool and using the
+:meth:`~scrapy.http.Request.from_curl()` method to generate an equivalent
+request::
+
+    from scrapy import Request
+
+    request = Request.from_curl(
+        "curl 'http://quotes.toscrape.com/api/quotes?page=1' -H 'User-Agent: Mozil"
+        "la/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0' -H 'Acce"
+        "pt: */*' -H 'Accept-Language: ca,en-US;q=0.7,en;q=0.3' --compressed -H 'X"
+        "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM"
+        "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW"
+        "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http"
+        "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
+
+Alternatively, if you want to know the arguments needed to recreate that
+request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
+function to get a dictionary with the equivalent arguments.
+
+As you can see, with a few inspections in the `Network`-tool we
 were able to easily replicate the dynamic requests of the scrolling 
 functionality of the page. Crawling dynamic pages can be quite
 daunting and pages can be very complex, but it (mostly) boils down
@@ -262,7 +300,7 @@ to identifying the correct request and replicating it in your spider.
 
 .. _Developer Tools: https://en.wikipedia.org/wiki/Web_development_tools
 .. _quotes.toscrape.com: http://quotes.toscrape.com
-.. _quotes.toscrape.com/scroll: quotes.toscrape.com/scroll/
+.. _quotes.toscrape.com/scroll: http://quotes.toscrape.com/scroll
 .. _quotes.toscrape.com/api/quotes?page=10: http://quotes.toscrape.com/api/quotes?page=10
 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
 

docs/topics/downloader-middleware.rst

@@ -41,7 +41,7 @@ previous (or subsequent) middleware being applied.
 
 If you want to disable a built-in middleware (the ones defined in
 :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it
-in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign `None`
+in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None``
 as its value.  For example, if you want to disable the user-agent middleware::
 
     DOWNLOADER_MIDDLEWARES = {
@@ -52,11 +52,17 @@ as its value.  For example, if you want to disable the user-agent middleware::
 Finally, keep in mind that some middlewares may need to be enabled through a
 particular setting. See each middleware documentation for more info.
 
+.. _topics-downloader-middleware-custom:
+
 Writing your own downloader middleware
 ======================================
 
-Each middleware component is a Python class that defines one or
-more of the following methods:
+Each downloader middleware is a Python class that defines one or more of the
+methods defined below.
+
+The main entry point is the ``from_crawler`` class method, which receives a
+:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
+object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
 .. module:: scrapy.downloadermiddlewares
 
@@ -193,7 +199,7 @@ CookiesMiddleware
 
    This middleware enables working with sites that require cookies, such as
    those that use sessions. It keeps track of cookies sent by web servers, and
-   send them back on subsequent requests (from that spider), just like web
+   sends them back on subsequent requests (from that spider), just like web
    browsers do.
 
 The following settings can be used to configure the cookie middleware:
@@ -253,8 +259,8 @@ COOKIES_DEBUG
 
 Default: ``False``
 
-If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie``
-header) and all cookies received in responses (ie. ``Set-Cookie`` header).
+If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie``
+header) and all cookies received in responses (i.e. ``Set-Cookie`` header).
 
 Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
 
@@ -342,10 +348,9 @@ HttpCacheMiddleware
 
         * :ref:`httpcache-storage-fs`
         * :ref:`httpcache-storage-dbm`
-        * :ref:`httpcache-storage-leveldb`
 
     You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
-    setting. Or you can also implement your own storage backend.
+    setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
 
     Scrapy ships with two HTTP cache policies:
 
@@ -357,26 +362,27 @@ HttpCacheMiddleware
 
     .. reqmeta:: dont_cache
 
-    You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals `True`.
+    You can also avoid caching a response on every policy using :reqmeta:`dont_cache` meta key equals ``True``.
+
+.. module:: scrapy.extensions.httpcache
+   :noindex:
 
 .. _httpcache-policy-dummy:
 
 Dummy policy (default)
 ~~~~~~~~~~~~~~~~~~~~~~
 
-This policy has no awareness of any HTTP Cache-Control directives.
-Every request and its corresponding response are cached.  When the same
-request is seen again, the response is returned without transferring
-anything from the Internet.
+.. class:: DummyPolicy
 
-The Dummy policy is useful for testing spiders faster (without having
-to wait for downloads every time) and for trying your spider offline,
-when an Internet connection is not available. The goal is to be able to
-"replay" a spider run *exactly as it ran before*.
+    This policy has no awareness of any HTTP Cache-Control directives.
+    Every request and its corresponding response are cached.  When the same
+    request is seen again, the response is returned without transferring
+    anything from the Internet.
 
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.DummyPolicy``
+    The Dummy policy is useful for testing spiders faster (without having
+    to wait for downloads every time) and for trying your spider offline,
+    when an Internet connection is not available. The goal is to be able to
+    "replay" a spider run *exactly as it ran before*.
 
 
 .. _httpcache-policy-rfc2616:
@@ -384,45 +390,44 @@ In order to use this policy, set:
 RFC2616 policy
 ~~~~~~~~~~~~~~
 
-This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
-Cache-Control awareness, aimed at production and used in continuous
-runs to avoid downloading unmodified data (to save bandwidth and speed up crawls).
+.. class:: RFC2616Policy
 
-what is implemented:
+    This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP
+    Cache-Control awareness, aimed at production and used in continuous
+    runs to avoid downloading unmodified data (to save bandwidth and speed up
+    crawls).
 
-* Do not attempt to store responses/requests with `no-store` cache-control directive set
-* Do not serve responses from cache if `no-cache` cache-control directive is set even for fresh responses
-* Compute freshness lifetime from `max-age` cache-control directive
-* Compute freshness lifetime from `Expires` response header
-* Compute freshness lifetime from `Last-Modified` response header (heuristic used by Firefox)
-* Compute current age from `Age` response header
-* Compute current age from `Date` header
-* Revalidate stale responses based on `Last-Modified` response header
-* Revalidate stale responses based on `ETag` response header
-* Set `Date` header for any received response missing it
-* Support `max-stale` cache-control directive in requests
+    What is implemented:
 
-  This allows spiders to be configured with the full RFC2616 cache policy,
-  but avoid revalidation on a request-by-request basis, while remaining
-  conformant with the HTTP spec.
+    * Do not attempt to store responses/requests with ``no-store`` cache-control directive set
+    * Do not serve responses from cache if ``no-cache`` cache-control directive is set even for fresh responses
+    * Compute freshness lifetime from ``max-age`` cache-control directive
+    * Compute freshness lifetime from ``Expires`` response header
+    * Compute freshness lifetime from ``Last-Modified`` response header (heuristic used by Firefox)
+    * Compute current age from ``Age`` response header
+    * Compute current age from ``Date`` header
+    * Revalidate stale responses based on ``Last-Modified`` response header
+    * Revalidate stale responses based on ``ETag`` response header
+    * Set ``Date`` header for any received response missing it
+    * Support ``max-stale`` cache-control directive in requests
 
-  Example:
+    This allows spiders to be configured with the full RFC2616 cache policy,
+    but avoid revalidation on a request-by-request basis, while remaining
+    conformant with the HTTP spec.
 
-  Add `Cache-Control: max-stale=600` to Request headers to accept responses that
-  have exceeded their expiration time by no more than 600 seconds.
+    Example:
 
-  See also: RFC2616, 14.9.3
+    Add ``Cache-Control: max-stale=600`` to Request headers to accept responses that
+    have exceeded their expiration time by no more than 600 seconds.
 
-what is missing:
+    See also: RFC2616, 14.9.3
 
-* `Pragma: no-cache` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
-* `Vary` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
-* Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
-* ... probably others ..
+    What is missing:
 
-In order to use this policy, set:
-
-* :setting:`HTTPCACHE_POLICY` to ``scrapy.extensions.httpcache.RFC2616Policy``
+    * ``Pragma: no-cache`` support https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
+    * ``Vary`` header support https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.6
+    * Invalidation after updates or deletes https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.10
+    * ... probably others ..
 
 
 .. _httpcache-storage-fs:
@@ -430,67 +435,102 @@ In order to use this policy, set:
 Filesystem storage backend (default)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-File system storage backend is available for the HTTP cache middleware.
+.. class:: FilesystemCacheStorage
 
-In order to use this storage backend, set:
+    File system storage backend is available for the HTTP cache middleware.
 
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.FilesystemCacheStorage``
+    Each request/response pair is stored in a different directory containing
+    the following files:
 
-Each request/response pair is stored in a different directory containing
-the following files:
+    *   ``request_body`` - the plain request body
 
- * ``request_body`` - the plain request body
- * ``request_headers`` - the request headers (in raw HTTP format)
- * ``response_body`` - the plain response body
- * ``response_headers`` - the request headers (in raw HTTP format)
- * ``meta`` - some metadata of this cache resource in Python ``repr()`` format
-   (grep-friendly format)
- * ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
-   efficient deserialization
+    *   ``request_headers`` - the request headers (in raw HTTP format)
 
-The directory name is made from the request fingerprint (see
-``scrapy.utils.request.fingerprint``), and one level of subdirectories is
-used to avoid creating too many files into the same directory (which is
-inefficient in many file systems). An example directory could be::
+    *   ``response_body`` - the plain response body
 
-   /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
+    *   ``response_headers`` - the request headers (in raw HTTP format)
+
+    *   ``meta`` - some metadata of this cache resource in Python ``repr()``
+        format (grep-friendly format)
+
+    *   ``pickled_meta`` - the same metadata in ``meta`` but pickled for more
+        efficient deserialization
+
+    The directory name is made from the request fingerprint (see
+    ``scrapy.utils.request.fingerprint``), and one level of subdirectories is
+    used to avoid creating too many files into the same directory (which is
+    inefficient in many file systems). An example directory could be::
+
+        /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
 
 .. _httpcache-storage-dbm:
 
 DBM storage backend
 ~~~~~~~~~~~~~~~~~~~
 
-.. versionadded:: 0.13
+.. class:: DbmCacheStorage
 
-A DBM_ storage backend is also available for the HTTP cache middleware.
+    .. versionadded:: 0.13
 
-By default, it uses the anydbm_ module, but you can change it with the
-:setting:`HTTPCACHE_DBM_MODULE` setting.
+    A DBM_ storage backend is also available for the HTTP cache middleware.
 
-In order to use this storage backend, set:
+    By default, it uses the :mod:`dbm`, but you can change it with the
+    :setting:`HTTPCACHE_DBM_MODULE` setting.
 
-* :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.DbmCacheStorage``
+.. _httpcache-storage-custom:
 
-.. _httpcache-storage-leveldb:
+Writing your own storage backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-LevelDB storage backend
-~~~~~~~~~~~~~~~~~~~~~~~
+You can implement a cache storage backend by creating a Python class that
+defines the methods described below.
 
-.. versionadded:: 0.23
+.. module:: scrapy.extensions.httpcache
 
-A LevelDB_ storage backend is also available for the HTTP cache middleware.
+.. class:: CacheStorage
 
-This backend is not recommended for development because only one process can
-access LevelDB databases at the same time, so you can't run a crawl and open
-the scrapy shell in parallel for the same spider.
+    .. method:: open_spider(spider)
 
-In order to use this storage backend:
+      This method gets called after a spider has been opened for crawling. It handles
+      the :signal:`open_spider <spider_opened>` signal.
 
-* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.extensions.httpcache.LeveldbCacheStorage``
-* install `LevelDB python bindings`_ like ``pip install leveldb``
+      :param spider: the spider which has been opened
+      :type spider: :class:`~scrapy.spiders.Spider` object
 
-.. _LevelDB: https://github.com/google/leveldb
-.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
+    .. method:: close_spider(spider)
+
+      This method gets called after a spider has been closed. It handles
+      the :signal:`close_spider <spider_closed>` signal.
+
+      :param spider: the spider which has been closed
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+    .. method:: retrieve_response(spider, request)
+
+      Return response if present in cache, or ``None`` otherwise.
+
+      :param spider: the spider which generated the request
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+      :param request: the request to find cached response for
+      :type request: :class:`~scrapy.http.Request` object
+
+    .. method:: store_response(spider, request, response)
+
+      Store the given response in the cache.
+
+      :param spider: the spider for which the response is intended
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+      :param request: the corresponding request the spider generated
+      :type request: :class:`~scrapy.http.Request` object
+
+      :param response: the response to store in the cache
+      :type response: :class:`~scrapy.http.Response` object
+
+In order to use your storage backend, set:
+
+* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class.
 
 
 HTTPCache middleware settings
@@ -586,7 +626,7 @@ HTTPCACHE_DBM_MODULE
 
 .. versionadded:: 0.13
 
-Default: ``'anydbm'``
+Default: ``'dbm'``
 
 The database module to use in the :ref:`DBM storage backend
 <httpcache-storage-dbm>`. This setting is specific to the DBM backend.
@@ -626,13 +666,13 @@ Default: ``False``
 If enabled, will cache pages unconditionally.
 
 A spider may wish to have all responses available in the cache, for
-future use with `Cache-Control: max-stale`, for instance. The
+future use with ``Cache-Control: max-stale``, for instance. The
 DummyPolicy caches all responses but never revalidates them, and
 sometimes a more nuanced policy is desirable.
 
-This setting still respects `Cache-Control: no-store` directives in responses.
-If you don't want that, filter `no-store` out of the Cache-Control headers in
-responses you feedto the cache middleware.
+This setting still respects ``Cache-Control: no-store`` directives in responses.
+If you don't want that, filter ``no-store`` out of the Cache-Control headers in
+responses you feed to the cache middleware.
 
 .. setting:: HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS
 
@@ -646,7 +686,7 @@ Default: ``[]``
 List of Cache-Control directives in responses to be ignored.
 
 Sites often set "no-store", "no-cache", "must-revalidate", etc., but get
-upset at the traffic a spider can generate if it respects those
+upset at the traffic a spider can generate if it actually respects those
 directives. This allows to selectively ignore Cache-Control directives
 that are known to be unimportant for the sites being crawled.
 
@@ -669,7 +709,7 @@ HttpCompressionMiddleware
    provided `brotlipy`_ is installed.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
-.. _brotlipy: https://pypi.python.org/pypi/brotlipy
+.. _brotlipy: https://pypi.org/project/brotlipy/
 
 HttpCompressionMiddleware Settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -729,6 +769,17 @@ RedirectMiddleware
 The urls which the request goes through (while being redirected) can be found
 in the ``redirect_urls`` :attr:`Request.meta <scrapy.http.Request.meta>` key.
 
+.. reqmeta:: redirect_reasons
+
+The reason behind each redirect in :reqmeta:`redirect_urls` can be found in the
+``redirect_reasons`` :attr:`Request.meta <scrapy.http.Request.meta>` key. For
+example: ``[301, 302, 307, 'meta refresh']``.
+
+The format of a reason depends on the middleware that handled the corresponding
+redirect. For example, :class:`RedirectMiddleware` indicates the triggering
+response status code as an integer, while :class:`MetaRefreshMiddleware`
+always uses the ``'meta refresh'`` string as reason.
+
 The :class:`RedirectMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
@@ -790,10 +841,12 @@ The :class:`MetaRefreshMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
 * :setting:`METAREFRESH_ENABLED`
+* :setting:`METAREFRESH_IGNORE_TAGS`
 * :setting:`METAREFRESH_MAXDELAY`
 
-This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`
-and :reqmeta:`redirect_urls` request meta keys as described for :class:`RedirectMiddleware`
+This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
+:reqmeta:`redirect_urls` and :reqmeta:`redirect_reasons` request meta keys as described
+for :class:`RedirectMiddleware`
 
 
 MetaRefreshMiddleware settings
@@ -810,6 +863,19 @@ Default: ``True``
 
 Whether the Meta Refresh middleware will be enabled.
 
+.. setting:: METAREFRESH_IGNORE_TAGS
+
+METAREFRESH_IGNORE_TAGS
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``[]``
+
+Meta tags within these tags are ignored.
+
+.. versionchanged:: 2.0
+   The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
+   ``['script', 'noscript']`` to ``[]``.
+
 .. setting:: METAREFRESH_MAXDELAY
 
 METAREFRESH_MAXDELAY
@@ -834,8 +900,6 @@ RetryMiddleware
 
 Failed pages are collected on the scraping process and rescheduled at the
 end, once the spider has finished crawling all regular (non failed) pages.
-Once there are no more failed pages to retry, this middleware sends a signal
-(retry_complete), so other extensions could connect to that signal.
 
 The :class:`RetryMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
@@ -882,7 +946,7 @@ precedence over the :setting:`RETRY_TIMES` setting.
 RETRY_HTTP_CODES
 ^^^^^^^^^^^^^^^^
 
-Default: ``[500, 502, 503, 504, 522, 524, 408]``
+Default: ``[500, 502, 503, 504, 522, 524, 408, 429]``
 
 Which HTTP response codes to retry. Other errors (DNS lookup issues,
 connections lost, etc) are always retried.
@@ -908,6 +972,24 @@ RobotsTxtMiddleware
     To make sure Scrapy respects robots.txt make sure the middleware is enabled
     and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
 
+    The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
+    user agent string to use for matching in the robots.txt_ file. If it
+    is ``None``, the User-Agent header you are sending with the request or the
+    :setting:`USER_AGENT` setting (in that order) will be used for determining
+    the user agent to use in the robots.txt_ file.
+
+    This middleware has to be combined with a robots.txt_ parser.
+
+    Scrapy ships with support for the following robots.txt_ parsers:
+
+    * :ref:`Protego <protego-parser>` (default)
+    * :ref:`RobotFileParser <python-robotfileparser>`
+    * :ref:`Reppy <reppy-parser>`
+    * :ref:`Robotexclusionrulesparser <rerp-parser>`
+
+    You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+    setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
+
 .. reqmeta:: dont_obey_robotstxt
 
 If :attr:`Request.meta <scrapy.http.Request.meta>` has
@@ -915,6 +997,129 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` has
 the request will be ignored by this middleware even if
 :setting:`ROBOTSTXT_OBEY` is enabled.
 
+Parsers vary in several aspects:
+
+* Language of implementation
+
+* Supported specification
+
+* Support for wildcard matching
+
+* Usage of `length based rule <https://developers.google.com/search/reference/robots_txt#order-of-precedence-for-group-member-lines>`_:
+  in particular for ``Allow`` and ``Disallow`` directives, where the most
+  specific rule based on the length of the path trumps the less specific
+  (shorter) rule
+
+Performance comparison of different parsers is available at `the following link
+<https://anubhavp28.github.io/gsoc-weekly-checkin-12/>`_.
+
+.. _protego-parser:
+
+Protego parser
+~~~~~~~~~~~~~~
+
+Based on `Protego <https://github.com/scrapy/protego>`_:
+
+* implemented in Python
+
+* is compliant with `Google's Robots.txt Specification
+  <https://developers.google.com/search/reference/robots_txt>`_
+
+* supports wildcard matching
+
+* uses the length based rule
+
+Scrapy uses this parser by default.
+
+.. _python-robotfileparser:
+
+RobotFileParser
+~~~~~~~~~~~~~~~
+
+Based on `RobotFileParser
+<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
+
+* is Python's built-in robots.txt_ parser
+
+* is compliant with `Martijn Koster's 1996 draft specification
+  <https://www.robotstxt.org/norobots-rfc.txt>`_
+
+* lacks support for wildcard matching
+
+* doesn't use the length based rule
+
+It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
+
+In order to use this parser, set:
+
+* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
+
+.. _reppy-parser:
+
+Reppy parser
+~~~~~~~~~~~~
+
+Based on `Reppy <https://github.com/seomoz/reppy/>`_:
+
+* is a Python wrapper around `Robots Exclusion Protocol Parser for C++
+  <https://github.com/seomoz/rep-cpp>`_
+
+* is compliant with `Martijn Koster's 1996 draft specification
+  <https://www.robotstxt.org/norobots-rfc.txt>`_
+
+* supports wildcard matching
+
+* uses the length based rule
+
+Native implementation, provides better speed than Protego.
+
+In order to use this parser:
+
+* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.ReppyRobotParser``
+
+.. _rerp-parser:
+
+Robotexclusionrulesparser
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Based on `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_:
+
+* implemented in Python
+
+* is compliant with `Martijn Koster's 1996 draft specification
+  <https://www.robotstxt.org/norobots-rfc.txt>`_
+
+* supports wildcard matching
+
+* doesn't use the length based rule
+
+In order to use this parser:
+
+* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
+  ``pip install robotexclusionrulesparser``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.RerpRobotParser``
+
+.. _support-for-new-robots-parser:
+
+Implementing support for a new parser
+-------------------------------------
+
+You can implement support for a new robots.txt_ parser by subclassing
+the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
+implementing the methods described below.
+
+.. module:: scrapy.robotstxt
+   :synopsis: robots.txt parser interface and implementations
+
+.. autoclass:: RobotParser
+   :members:
+
+.. _robots.txt: https://www.robotstxt.org/
 
 DownloaderStats
 ---------------
@@ -940,7 +1145,7 @@ UserAgentMiddleware
 
    Middleware that allows spiders to override the default user agent.
 
-   In order for a spider to override the default user agent, its `user_agent`
+   In order for a spider to override the default user agent, its ``user_agent``
    attribute must be set.
 
 .. _ajaxcrawl-middleware:
@@ -954,7 +1159,7 @@ AjaxCrawlMiddleware
 
    Middleware that finds 'AJAX crawlable' page variants based
    on meta-fragment html tag. See
-   https://developers.google.com/webmasters/ajax-crawling/docs/getting-started
+   https://developers.google.com/search/docs/ajax-crawling/docs/getting-started
    for more info.
 
    .. note::
@@ -1001,4 +1206,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
 
 
 .. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html

docs/topics/dynamic-content.rst

@@ -0,0 +1,253 @@
+.. _topics-dynamic-content:
+
+====================================
+Selecting dynamically-loaded content
+====================================
+
+Some webpages show the desired data when you load them in a web browser.
+However, when you download them using Scrapy, you cannot reach the desired data
+using :ref:`selectors <topics-selectors>`.
+
+When this happens, the recommended approach is to
+:ref:`find the data source <topics-finding-data-source>` and extract the data
+from it.
+
+If you fail to do that, and you can nonetheless access the desired data through
+the :ref:`DOM <topics-livedom>` from your web browser, see
+:ref:`topics-javascript-rendering`.
+
+.. _topics-finding-data-source:
+
+Finding the data source
+=======================
+
+To extract the desired data, you must first find its source location.
+
+If the data is in a non-text-based format, such as an image or a PDF document,
+use the :ref:`network tool <topics-network-tool>` of your web browser to find
+the corresponding request, and :ref:`reproduce it
+<topics-reproducing-requests>`.
+
+If your web browser lets you select the desired data as text, the data may be
+defined in embedded JavaScript code, or loaded from an external resource in a
+text-based format.
+
+In that case, you can use a tool like wgrep_ to find the URL of that resource.
+
+If the data turns out to come from the original URL itself, you must
+:ref:`inspect the source code of the webpage <topics-inspecting-source>` to
+determine where the data is located.
+
+If the data comes from a different URL, you will need to :ref:`reproduce the
+corresponding request <topics-reproducing-requests>`.
+
+.. _topics-inspecting-source:
+
+Inspecting the source code of a webpage
+=======================================
+
+Sometimes you need to inspect the source code of a webpage (not the
+:ref:`DOM <topics-livedom>`) to determine where some desired data is located.
+
+Use Scrapy’s :command:`fetch` command to download the webpage contents as seen
+by Scrapy::
+
+    scrapy fetch --nolog https://example.com > response.html
+
+If the desired data is in embedded JavaScript code within a ``<script/>``
+element, see :ref:`topics-parsing-javascript`.
+
+If you cannot find the desired data, first make sure it’s not just Scrapy:
+download the webpage with an HTTP client like curl_ or wget_ and see if the
+information can be found in the response they get.
+
+If they get a response with the desired data, modify your Scrapy
+:class:`~scrapy.http.Request` to match that of the other HTTP client. For
+example, try using the same user-agent string (:setting:`USER_AGENT`) or the
+same :attr:`~scrapy.http.Request.headers`.
+
+If they also get a response without the desired data, you’ll need to take
+steps to make your request more similar to that of the web browser. See
+:ref:`topics-reproducing-requests`.
+
+.. _topics-reproducing-requests:
+
+Reproducing requests
+====================
+
+Sometimes we need to reproduce a request the way our web browser performs it.
+
+Use the :ref:`network tool <topics-network-tool>` of your web browser to see
+how your web browser performs the desired request, and try to reproduce that
+request with Scrapy.
+
+It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
+method and URL. However, you may also need to reproduce the body, headers and
+form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
+
+As all major browsers allow to export the requests in `cURL
+<https://curl.haxx.se/>`_ format, Scrapy incorporates the method
+:meth:`~scrapy.http.Request.from_curl()` to generate an equivalent
+:class:`~scrapy.http.Request` from a cURL command. To get more information
+visit :ref:`request from curl <requests-from-curl>` inside the network
+tool section.
+
+Once you get the expected response, you can :ref:`extract the desired data from
+it <topics-handling-response-formats>`.
+
+You can reproduce any request with Scrapy. However, some times reproducing all
+necessary requests may not seem efficient in developer time. If that is your
+case, and crawling speed is not a major concern for you, you can alternatively
+consider :ref:`JavaScript pre-rendering <topics-javascript-rendering>`.
+
+If you get the expected response `sometimes`, but not always, the issue is
+probably not your request, but the target server. The target server might be
+buggy, overloaded, or :ref:`banning <bans>` some of your requests.
+
+.. _topics-handling-response-formats:
+
+Handling different response formats
+===================================
+
+Once you have a response with the desired data, how you extract the desired
+data from it depends on the type of response:
+
+-   If the response is HTML or XML, use :ref:`selectors
+    <topics-selectors>` as usual.
+
+-   If the response is JSON, use `json.loads`_ to load the desired data from
+    :attr:`response.text <scrapy.http.TextResponse.text>`::
+
+        data = json.loads(response.text)
+
+    If the desired data is inside HTML or XML code embedded within JSON data,
+    you can load that HTML or XML code into a
+    :class:`~scrapy.selector.Selector` and then
+    :ref:`use it <topics-selectors>` as usual::
+
+        selector = Selector(data['html'])
+
+-   If the response is JavaScript, or HTML with a ``<script/>`` element
+    containing the desired data, see :ref:`topics-parsing-javascript`.
+
+-   If the response is CSS, use a `regular expression`_ to extract the desired
+    data from :attr:`response.text <scrapy.http.TextResponse.text>`.
+
+.. _topics-parsing-images:
+
+-   If the response is an image or another format based on images (e.g. PDF),
+    read the response as bytes from
+    :attr:`response.body <scrapy.http.TextResponse.body>` and use an OCR
+    solution to extract the desired data as text.
+
+    For example, you can use pytesseract_. To read a table from a PDF,
+    `tabula-py`_ may be a better choice.
+
+-   If the response is SVG, or HTML with embedded SVG containing the desired
+    data, you may be able to extract the desired data using
+    :ref:`selectors <topics-selectors>`, since SVG is based on XML.
+
+    Otherwise, you might need to convert the SVG code into a raster image, and
+    :ref:`handle that raster image <topics-parsing-images>`.
+
+.. _topics-parsing-javascript:
+
+Parsing JavaScript code
+=======================
+
+If the desired data is hardcoded in JavaScript, you first need to get the
+JavaScript code:
+
+-   If the JavaScript code is in a JavaScript file, simply read
+    :attr:`response.text <scrapy.http.TextResponse.text>`.
+
+-   If the JavaScript code is within a ``<script/>`` element of an HTML page,
+    use :ref:`selectors <topics-selectors>` to extract the text within that
+    ``<script/>`` element.
+
+Once you have a string with the JavaScript code, you can extract the desired
+data from it:
+
+-   You might be able to use a `regular expression`_ to extract the desired
+    data in JSON format, which you can then parse with `json.loads`_.
+
+    For example, if the JavaScript code contains a separate line like
+    ``var data = {"field": "value"};`` you can extract that data as follows:
+
+    >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
+    >>> json_data = response.css('script::text').re_first(pattern)
+    >>> json.loads(json_data)
+    {'field': 'value'}
+
+-   Otherwise, use js2xml_ to convert the JavaScript code into an XML document
+    that you can parse using :ref:`selectors <topics-selectors>`.
+
+    For example, if the JavaScript code contains
+    ``var data = {field: "value"};`` you can extract that data as follows:
+
+    >>> import js2xml
+    >>> import lxml.etree
+    >>> from parsel import Selector
+    >>> javascript = response.css('script::text').get()
+    >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
+    >>> selector = Selector(text=xml)
+    >>> selector.css('var[name="data"]').get()
+    '<var name="data"><object><property name="field"><string>value</string></property></object></var>'
+
+.. _topics-javascript-rendering:
+
+Pre-rendering JavaScript
+========================
+
+On webpages that fetch data from additional requests, reproducing those
+requests that contain the desired data is the preferred approach. The effort is
+often worth the result: structured, complete data with minimum parsing time and
+network transfer.
+
+However, sometimes it can be really hard to reproduce certain requests. Or you
+may need something that no request can give you, such as a screenshot of a
+webpage as seen in a web browser.
+
+In these cases use the Splash_ JavaScript-rendering service, along with
+`scrapy-splash`_ for seamless integration.
+
+Splash returns as HTML the :ref:`DOM <topics-livedom>` of a webpage, so that
+you can parse it with :ref:`selectors <topics-selectors>`. It provides great
+flexibility through configuration_ or scripting_.
+
+If you need something beyond what Splash offers, such as interacting with the
+DOM on-the-fly from Python code instead of using a previously-written script,
+or handling multiple web browser windows, you might need to
+:ref:`use a headless browser <topics-headless-browsing>` instead.
+
+.. _configuration: https://splash.readthedocs.io/en/stable/api.html
+.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html
+
+.. _topics-headless-browsing:
+
+Using a headless browser
+========================
+
+A `headless browser`_ is a special web browser that provides an API for
+automation.
+
+The easiest way to use a headless browser with Scrapy is to use Selenium_,
+along with `scrapy-selenium`_ for seamless integration.
+
+
+.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
+.. _curl: https://curl.haxx.se/
+.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
+.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
+.. _js2xml: https://github.com/scrapinghub/js2xml
+.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
+.. _pytesseract: https://github.com/madmaze/pytesseract
+.. _regular expression: https://docs.python.org/3/library/re.html
+.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+.. _Selenium: https://www.selenium.dev/
+.. _Splash: https://github.com/scrapinghub/splash
+.. _tabula-py: https://github.com/chezou/tabula-py
+.. _wget: https://www.gnu.org/software/wget/
+.. _wgrep: https://github.com/stav/wgrep

docs/topics/email.rst

@@ -9,19 +9,19 @@ Sending e-mail
 
 Although Python makes sending e-mails relatively easy via the `smtplib`_
 library, Scrapy provides its own facility for sending e-mails which is very
-easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
-interfering with the non-blocking IO of the crawler. It also provides a
-simple API for sending attachments and it's very easy to configure, with a few
-:ref:`settings <topics-email-settings>`.
+easy to use and it's implemented using :doc:`Twisted non-blocking IO
+<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
+IO of the crawler. It also provides a simple API for sending attachments and
+it's very easy to configure, with a few :ref:`settings
+<topics-email-settings>`.
 
 .. _smtplib: https://docs.python.org/2/library/smtplib.html
-.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
 
 Quick example
 =============
 
 There are two ways to instantiate the mail sender. You can instantiate it using
-the standard constructor::
+the standard ``__init__`` method::
 
     from scrapy.mail import MailSender
     mailer = MailSender()
@@ -39,7 +39,8 @@ MailSender class reference
 ==========================
 
 MailSender is the preferred class to use for sending emails from Scrapy, as it
-uses `Twisted non-blocking IO`_, like the rest of the framework.
+uses :doc:`Twisted non-blocking IO <twisted:core/howto/defer-intro>`, like the
+rest of the framework.
 
 .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
 
@@ -111,7 +112,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
 Mail settings
 =============
 
-These settings define the default constructor values of the :class:`MailSender`
+These settings define the default ``__init__`` method values of the :class:`MailSender`
 class, and can be used to configure e-mail notifications in your project without
 writing any code (for those extensions and code that uses :class:`MailSender`).
 

docs/topics/exporters.rst

@@ -42,7 +42,7 @@ value of one of their fields::
 
     from scrapy.exporters import XmlItemExporter
 
-    class PerYearXmlExportPipeline(object):
+    class PerYearXmlExportPipeline:
         """Distribute items across multiple XML files according to their 'year' field"""
 
         def open_spider(self, spider):
@@ -51,7 +51,6 @@ value of one of their fields::
         def close_spider(self, spider):
             for exporter in self.year_to_exporter.values():
                 exporter.finish_exporting()
-                exporter.file.close()
 
         def _exporter_for_item(self, item):
             year = item['year']
@@ -88,8 +87,8 @@ described next.
 1. Declaring a serializer in the field
 --------------------------------------
 
-If you use :class:`~.Item` you can declare a serializer in the 
-:ref:`field metadata <topics-items-fields>`. The serializer must be 
+If you use :class:`~.Item` you can declare a serializer in the
+:ref:`field metadata <topics-items-fields>`. The serializer must be
 a callable which receives a value and returns its serialized form.
 
 Example::
@@ -138,17 +137,20 @@ output examples, which assume you're exporting these two items::
 BaseItemExporter
 ----------------
 
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
 
    This is the (abstract) base class for all Item Exporters. It provides
    support for common features used by all (concrete) Item Exporters, such as
    defining what fields to export, whether to export empty fields, or which
    encoding to use.
 
-   These features can be configured through the constructor arguments which
+   These features can be configured through the ``__init__`` method arguments which
    populate their respective instance attributes: :attr:`fields_to_export`,
    :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
 
+   .. versionadded:: 2.0
+      The *dont_fail* parameter.
+
    .. method:: export_item(item)
 
       Exports the given item. This method must be implemented in subclasses.
@@ -165,9 +167,9 @@ BaseItemExporter
       value unchanged except for ``unicode`` values which are encoded to
       ``str`` using the encoding declared in the :attr:`encoding` attribute.
 
-      :param field: the field being serialized. If a raw dict is being 
+      :param field: the field being serialized. If a raw dict is being
           exported (not :class:`~.Item`) *field* value is an empty dict.
-      :type field: :class:`~scrapy.item.Field` object or an empty dict 
+      :type field: :class:`~scrapy.item.Field` object or an empty dict
 
       :param name: the name of the field being serialized
       :type name: str
@@ -223,6 +225,12 @@ BaseItemExporter
       * ``indent<=0`` each item on its own line, no indentation
       * ``indent>0`` each item on its own line, indented with the provided numeric value
 
+PythonItemExporter
+------------------
+
+.. autoclass:: PythonItemExporter
+
+
 .. highlight:: none
 
 XmlItemExporter
@@ -241,8 +249,8 @@ XmlItemExporter
    :param item_element: The name of each item element in the exported XML.
    :type item_element: str
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor.
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method.
 
    A typical output of this exporter would be::
 
@@ -301,9 +309,9 @@ CsvItemExporter
       multi-valued fields, if found.
    :type include_headers_line: str
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor, and the leftover arguments to the
-   `csv.writer`_ constructor, so you can use any `csv.writer` constructor
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
+   `csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
    argument to customize this exporter.
 
    A typical output of this exporter would be::
@@ -329,8 +337,8 @@ PickleItemExporter
 
    For more information, refer to the `pickle module documentation`_.
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor.
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method.
 
    Pickle isn't a human readable format, so no output examples are provided.
 
@@ -346,8 +354,8 @@ PprintItemExporter
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
-   The additional keyword arguments of this constructor are passed to the
-   :class:`BaseItemExporter` constructor.
+   The additional keyword arguments of this ``__init__`` method are passed to the
+   :class:`BaseItemExporter` ``__init__`` method.
 
    A typical output of this exporter would be::
 
@@ -362,10 +370,10 @@ JsonItemExporter
 .. class:: JsonItemExporter(file, \**kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing all
-   objects as a list of objects. The additional constructor arguments are
-   passed to the :class:`BaseItemExporter` constructor, and the leftover
-   arguments to the `JSONEncoder`_ constructor, so you can use any
-   `JSONEncoder`_ constructor argument to customize this exporter.
+   objects as a list of objects. The additional ``__init__`` method arguments are
+   passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
+   arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
+   `JSONEncoder`_ ``__init__`` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -393,10 +401,10 @@ JsonLinesItemExporter
 .. class:: JsonLinesItemExporter(file, \**kwargs)
 
    Exports Items in JSON format to the specified file-like object, writing one
-   JSON-encoded item per line. The additional constructor arguments are passed
-   to the :class:`BaseItemExporter` constructor, and the leftover arguments to
-   the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
-   constructor argument to customize this exporter.
+   JSON-encoded item per line. The additional ``__init__`` method arguments are passed
+   to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
+   the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
+   ``__init__`` method argument to customize this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -410,3 +418,8 @@ JsonLinesItemExporter
    this exporter is well suited for serializing large amounts of data.
 
 .. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
+
+MarshalItemExporter
+-------------------
+
+.. autoclass:: MarshalItemExporter

docs/topics/extensions.rst

@@ -19,7 +19,7 @@ settings, just like any other Scrapy code.
 It is customary for extensions to prefix their settings with their own name, to
 avoid collision with existing (and future) extensions. For example, a
 hypothetic extension to handle `Google Sitemaps`_ would use settings like
-`GOOGLESITEMAP_ENABLED`, `GOOGLESITEMAP_DEPTH`, and so on.
+``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
 
 .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
 
@@ -28,7 +28,7 @@ Loading & activating extensions
 
 Extensions are loaded and activated at startup by instantiating a single
 instance of the extension class. Therefore, all the extension initialization
-code must be performed in the class constructor (``__init__`` method).
+code must be performed in the class ``__init__`` method.
 
 To make an extension available, add it to the :setting:`EXTENSIONS` setting in
 your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
@@ -63,7 +63,7 @@ but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set.
 Disabling an extension
 ======================
 
-In order to disable an extension that comes enabled by default (ie. those
+In order to disable an extension that comes enabled by default (i.e. those
 included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
 ``None``. For example::
 
@@ -107,7 +107,7 @@ Here is the code of such extension::
 
     logger = logging.getLogger(__name__)
 
-    class SpiderOpenCloseLogging(object):
+    class SpiderOpenCloseLogging:
 
         def __init__(self, item_count):
             self.item_count = item_count
@@ -183,7 +183,7 @@ Telnet console extension
 .. module:: scrapy.extensions.telnet
    :synopsis: Telnet console
 
-.. class:: scrapy.extensions.telnet.TelnetConsole
+.. class:: TelnetConsole
 
 Provides a telnet console for getting into a Python interpreter inside the
 currently running Scrapy process, which can be very useful for debugging.
@@ -200,7 +200,7 @@ Memory usage extension
 .. module:: scrapy.extensions.memusage
    :synopsis: Memory usage extension
 
-.. class:: scrapy.extensions.memusage.MemoryUsage
+.. class:: MemoryUsage
 
 .. note:: This extension does not work in Windows.
 
@@ -228,7 +228,7 @@ Memory debugger extension
 .. module:: scrapy.extensions.memdebug
    :synopsis: Memory debugger extension
 
-.. class:: scrapy.extensions.memdebug.MemoryDebugger
+.. class:: MemoryDebugger
 
 An extension for debugging memory usage. It collects information about:
 
@@ -244,7 +244,7 @@ Close spider extension
 .. module:: scrapy.extensions.closespider
    :synopsis: Close spider extension
 
-.. class:: scrapy.extensions.closespider.CloseSpider
+.. class:: CloseSpider
 
 Closes a spider automatically when some conditions are met, using a specific
 closing reason for each condition.
@@ -317,7 +317,7 @@ StatsMailer extension
 .. module:: scrapy.extensions.statsmailer
    :synopsis: StatsMailer extension
 
-.. class:: scrapy.extensions.statsmailer.StatsMailer
+.. class:: StatsMailer
 
 This simple extension can be used to send a notification e-mail every time a
 domain has finished scraping, including the Scrapy stats collected. The email
@@ -333,7 +333,7 @@ Debugging extensions
 Stack trace dump extension
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. class:: scrapy.extensions.debug.StackTraceDump
+.. class:: StackTraceDump
 
 Dumps information about the running process when a `SIGQUIT`_ or `SIGUSR2`_
 signal is received. The information dumped is the following:
@@ -345,7 +345,7 @@ signal is received. The information dumped is the following:
 After the stack trace and engine status is dumped, the Scrapy process continues
 running normally.
 
-This extension only works on POSIX-compliant platforms (ie. not Windows),
+This extension only works on POSIX-compliant platforms (i.e. not Windows),
 because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows.
 
 There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
@@ -362,15 +362,15 @@ There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
 Debugger extension
 ~~~~~~~~~~~~~~~~~~
 
-.. class:: scrapy.extensions.debug.Debugger
+.. class:: Debugger
 
 Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
 signal is received. After the debugger is exited, the Scrapy process continues
 running normally.
 
-For more info see `Debugging in Python`.
+For more info see `Debugging in Python`_.
 
-This extension only works on POSIX-compliant platforms (ie. not Windows).
+This extension only works on POSIX-compliant platforms (i.e. not Windows).
 
 .. _Python debugger: https://docs.python.org/2/library/pdb.html
 .. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/

docs/topics/feed-exports.rst

@@ -12,7 +12,7 @@ generating an "export file" with the scraped data (commonly called "export
 feed") to be consumed by other systems.
 
 Scrapy provides this functionality out of the box with the Feed Exports, which
-allows you to generate a feed with the scraped items, using multiple
+allows you to generate feeds with the scraped items, using multiple
 serialization formats and storage backends.
 
 .. _topics-feed-format:
@@ -36,7 +36,7 @@ But you can also extend the supported format through the
 JSON
 ----
 
- * :setting:`FEED_FORMAT`: ``json``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``json``
  * Exporter used: :class:`~scrapy.exporters.JsonItemExporter`
  * See :ref:`this warning <json-with-large-data>` if you're using JSON with
    large feeds.
@@ -46,7 +46,7 @@ JSON
 JSON lines
 ----------
 
- * :setting:`FEED_FORMAT`: ``jsonlines``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``jsonlines``
  * Exporter used: :class:`~scrapy.exporters.JsonLinesItemExporter`
 
 .. _topics-feed-format-csv:
@@ -54,7 +54,7 @@ JSON lines
 CSV
 ---
 
- * :setting:`FEED_FORMAT`: ``csv``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
  * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
  * To specify columns to export and their order use
    :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
@@ -66,7 +66,7 @@ CSV
 XML
 ---
 
- * :setting:`FEED_FORMAT`: ``xml``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``xml``
  * Exporter used: :class:`~scrapy.exporters.XmlItemExporter`
 
 .. _topics-feed-format-pickle:
@@ -74,7 +74,7 @@ XML
 Pickle
 ------
 
- * :setting:`FEED_FORMAT`: ``pickle``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``pickle``
  * Exporter used: :class:`~scrapy.exporters.PickleItemExporter`
 
 .. _topics-feed-format-marshal:
@@ -82,7 +82,7 @@ Pickle
 Marshal
 -------
 
- * :setting:`FEED_FORMAT`: ``marshal``
+ * Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
  * Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
 
 
@@ -91,20 +91,20 @@ Marshal
 Storages
 ========
 
-When using the feed exports you define where to store the feed using a URI_
-(through the :setting:`FEED_URI` setting). The feed exports supports multiple
+When using the feed exports you define where to store the feed using one or multiple URIs_
+(through the :setting:`FEEDS` setting). The feed exports supports multiple
 storage backend types which are defined by the URI scheme.
 
 The storages backends supported out of the box are:
 
  * :ref:`topics-feed-storage-fs`
  * :ref:`topics-feed-storage-ftp`
- * :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
+ * :ref:`topics-feed-storage-s3` (requires botocore_)
  * :ref:`topics-feed-storage-stdout`
 
 Some storage backends may be unavailable if the required external libraries are
 not available. For example, the S3 backend is only available if the botocore_
-or boto_ library is installed (Scrapy supports boto_ only on Python 2).
+library is installed.
 
 
 .. _topics-feed-uri-params:
@@ -164,6 +164,11 @@ The feeds are stored in a FTP server.
  * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
  * Required external libraries: none
 
+FTP supports two different connection modes: `active or passive
+<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
+mode by default. To use the active connection mode instead, set the
+:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
+
 .. _topics-feed-storage-s3:
 
 S3
@@ -177,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
    * ``s3://mybucket/path/to/export.csv``
    * ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
- * Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
+ * Required external libraries: `botocore`_
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -185,6 +190,10 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
+You can also define a custom ACL for exported feeds using this setting:
+
+ * :setting:`FEED_STORAGE_S3_ACL`
+
 .. _topics-feed-storage-stdout:
 
 Standard output
@@ -202,36 +211,66 @@ Settings
 
 These are the settings used for configuring the feed exports:
 
- * :setting:`FEED_URI` (mandatory)
- * :setting:`FEED_FORMAT`
- * :setting:`FEED_STORAGES`
- * :setting:`FEED_EXPORTERS`
- * :setting:`FEED_STORE_EMPTY`
+ * :setting:`FEEDS` (mandatory)
  * :setting:`FEED_EXPORT_ENCODING`
+ * :setting:`FEED_STORE_EMPTY`
  * :setting:`FEED_EXPORT_FIELDS`
  * :setting:`FEED_EXPORT_INDENT`
+ * :setting:`FEED_STORAGES`
+ * :setting:`FEED_STORAGE_FTP_ACTIVE`
+ * :setting:`FEED_STORAGE_S3_ACL`
+ * :setting:`FEED_EXPORTERS`
 
 .. currentmodule:: scrapy.extensions.feedexport
 
-.. setting:: FEED_URI
+.. setting:: FEEDS
 
-FEED_URI
---------
+FEEDS
+-----
 
-Default: ``None``
+.. versionadded:: 2.1
 
-The URI of the export feed. See :ref:`topics-feed-storage-backends` for
-supported URI schemes.
+Default: ``{}``
 
-This setting is required for enabling the feed exports.
+A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
+object) and each value is a nested dictionary containing configuration
+parameters for the specific feed.
+This setting is required for enabling the feed export feature.
 
-.. setting:: FEED_FORMAT
+See :ref:`topics-feed-storage-backends` for supported URI schemes.
 
-FEED_FORMAT
------------
+For instance::
 
-The serialization format to be used for the feed. See
-:ref:`topics-feed-format` for possible values.
+    {
+        'items.json': {
+            'format': 'json',
+            'encoding': 'utf8',
+            'store_empty': False,
+            'fields': None,
+            'indent': 4,
+        }, 
+        '/home/user/documents/items.xml': {
+            'format': 'xml',
+            'fields': ['name', 'price'],
+            'encoding': 'latin1',
+            'indent': 8,
+        },
+        pathlib.Path('items.csv'): {
+            'format': 'csv',
+            'fields': ['price', 'name'],
+        },
+    }
+
+The following is a list of the accepted keys and the setting that is used
+as a fallback value if that key is not provided for a specific feed definition.
+
+* ``format``: the serialization format to be used for the feed.
+  See :ref:`topics-feed-format` for possible values. 
+  Mandatory, no fallback setting
+* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
+* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
+* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
+* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
 
 .. setting:: FEED_EXPORT_ENCODING
 
@@ -290,7 +329,7 @@ FEED_STORE_EMPTY
 
 Default: ``False``
 
-Whether to export empty feeds (ie. feeds with no items).
+Whether to export empty feeds (i.e. feeds with no items).
 
 .. setting:: FEED_STORAGES
 
@@ -302,6 +341,30 @@ Default: ``{}``
 A dict containing additional feed storage backends supported by your project.
 The keys are URI schemes and the values are paths to storage classes.
 
+.. setting:: FEED_STORAGE_FTP_ACTIVE
+
+FEED_STORAGE_FTP_ACTIVE
+-----------------------
+
+Default: ``False``
+
+Whether to use the active connection mode when exporting feeds to an FTP server
+(``True``) or use the passive connection mode instead (``False``, default).
+
+For information about FTP connection modes, see `What is the difference between
+active and passive FTP? <https://stackoverflow.com/a/1699163>`_.
+
+.. setting:: FEED_STORAGE_S3_ACL
+
+FEED_STORAGE_S3_ACL
+-------------------
+
+Default: ``''`` (empty string)
+
+A string containing a custom ACL for feeds exported to Amazon S3 by your project.
+
+For a complete list of available values, access the `Canned ACL`_ section on Amazon S3 docs.
+
 .. setting:: FEED_STORAGES_BASE
 
 FEED_STORAGES_BASE
@@ -362,7 +425,7 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
         'csv': None,
     }
 
-.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
+.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
-.. _boto: https://github.com/boto/boto
 .. _botocore: https://github.com/boto/botocore
+.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl

docs/topics/item-pipeline.rst

@@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
 
    This method is called for every item pipeline component. :meth:`process_item`
    must either: return a dict with data, return an :class:`~scrapy.item.Item`
-   (or any descendant class) object, return a `Twisted Deferred`_ or raise
+   (or any descendant class) object, return a
+   :class:`~twisted.internet.defer.Deferred` or raise
    :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
    processed by further pipeline components.
 
@@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
    :type crawler: :class:`~scrapy.crawler.Crawler` object
 
 
-.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
-
 Item pipeline example
 =====================
 
@@ -82,13 +81,13 @@ contain a price::
 
     from scrapy.exceptions import DropItem
 
-    class PricePipeline(object):
+    class PricePipeline:
 
         vat_factor = 1.15
 
         def process_item(self, item, spider):
-            if item['price']:
-                if item['price_excludes_vat']:
+            if item.get('price'):
+                if item.get('price_excludes_vat'):
                     item['price'] = item['price'] * self.vat_factor
                 return item
             else:
@@ -104,7 +103,7 @@ format::
 
    import json
 
-   class JsonWriterPipeline(object):
+   class JsonWriterPipeline:
 
        def open_spider(self, spider):
            self.file = open('items.jl', 'w')
@@ -133,7 +132,7 @@ method and how to clean up the resources properly.::
 
     import pymongo
 
-    class MongoPipeline(object):
+    class MongoPipeline:
 
         collection_name = 'scrapy_items'
 
@@ -159,17 +158,20 @@ method and how to clean up the resources properly.::
             self.db[self.collection_name].insert_one(dict(item))
             return item
 
-.. _MongoDB: https://www.mongodb.org/
-.. _pymongo: https://api.mongodb.org/python/current/
+.. _MongoDB: https://www.mongodb.com/
+.. _pymongo: https://api.mongodb.com/python/current/
 
 
+.. _ScreenshotPipeline:
+
 Take screenshot of item
 -----------------------
 
-This example demonstrates how to return Deferred_ from :meth:`process_item` method.
+This example demonstrates how to return a
+:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
 It uses Splash_ to render screenshot of item url. Pipeline
-makes request to locally running instance of Splash_. After request is downloaded
-and Deferred callback fires, it saves item to a file and adds filename to an item.
+makes request to locally running instance of Splash_. After request is downloaded,
+it saves the screenshot to a file and adds filename to the item.
 
 ::
 
@@ -178,21 +180,18 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
     from urllib.parse import quote
 
 
-    class ScreenshotPipeline(object):
+    class ScreenshotPipeline:
         """Pipeline that uses Splash to render screenshot of
         every Scrapy item."""
 
         SPLASH_URL = "http://localhost:8050/render.png?url={}"
 
-        def process_item(self, item, spider):
+        async def process_item(self, item, spider):
             encoded_item_url = quote(item["url"])
             screenshot_url = self.SPLASH_URL.format(encoded_item_url)
             request = scrapy.Request(screenshot_url)
-            dfd = spider.crawler.engine.download(request, spider)
-            dfd.addBoth(self.return_item, item)
-            return dfd
+            response = await spider.crawler.engine.download(request, spider)
 
-        def return_item(self, response, item):
             if response.status != 200:
                 # Error happened, return item.
                 return item
@@ -209,7 +208,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
             return item
 
 .. _Splash: https://splash.readthedocs.io/en/stable/
-.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
 
 Duplicates filter
 -----------------
@@ -221,7 +219,7 @@ returns multiples items with the same id::
 
     from scrapy.exceptions import DropItem
 
-    class DuplicatesPipeline(object):
+    class DuplicatesPipeline:
 
         def __init__(self):
             self.ids_seen = set()

docs/topics/items.rst

@@ -16,12 +16,12 @@ especially in a larger project with many spiders.
 To define common output data format Scrapy provides the :class:`Item` class.
 :class:`Item` objects are simple containers used to collect the scraped data.
 They provide a `dictionary-like`_ API with a convenient syntax for declaring
-their available fields. 
+their available fields.
 
-Various Scrapy components use extra information provided by Items: 
+Various Scrapy components use extra information provided by Items:
 exporters look at declared fields to figure out columns to export,
 serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help find memory leaks 
+tracks Item instances to help find memory leaks
 (see :ref:`topics-leaks-trackrefs`), etc.
 
 .. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
@@ -40,6 +40,7 @@ objects. Here is an example::
         name = scrapy.Field()
         price = scrapy.Field()
         stock = scrapy.Field()
+        tags = scrapy.Field()
         last_updated = scrapy.Field(serializer=str)
 
 .. note:: Those familiar with `Django`_ will notice that Scrapy Items are
@@ -83,105 +84,128 @@ notice the API is very similar to the `dict API`_.
 Creating items
 --------------
 
-::
+>>> product = Product(name='Desktop PC', price=1000)
+>>> print(product)
+Product(name='Desktop PC', price=1000)
 
-    >>> product = Product(name='Desktop PC', price=1000)
-    >>> print(product)
-    Product(name='Desktop PC', price=1000)
 
 Getting field values
 --------------------
 
-::
+>>> product['name']
+Desktop PC
+>>> product.get('name')
+Desktop PC
 
-    >>> product['name']
-    Desktop PC
-    >>> product.get('name')
-    Desktop PC
+>>> product['price']
+1000
 
-    >>> product['price']
-    1000
+>>> product['last_updated']
+Traceback (most recent call last):
+    ...
+KeyError: 'last_updated'
 
-    >>> product['last_updated']
-    Traceback (most recent call last):
-        ...
-    KeyError: 'last_updated'
+>>> product.get('last_updated', 'not set')
+not set
 
-    >>> product.get('last_updated', 'not set')
-    not set
+>>> product['lala'] # getting unknown field
+Traceback (most recent call last):
+    ...
+KeyError: 'lala'
 
-    >>> product['lala'] # getting unknown field
-    Traceback (most recent call last):
-        ...
-    KeyError: 'lala'
+>>> product.get('lala', 'unknown field')
+'unknown field'
 
-    >>> product.get('lala', 'unknown field')
-    'unknown field'
+>>> 'name' in product  # is name field populated?
+True
 
-    >>> 'name' in product  # is name field populated?
-    True
+>>> 'last_updated' in product  # is last_updated populated?
+False
 
-    >>> 'last_updated' in product  # is last_updated populated?
-    False
+>>> 'last_updated' in product.fields  # is last_updated a declared field?
+True
 
-    >>> 'last_updated' in product.fields  # is last_updated a declared field?
-    True
+>>> 'lala' in product.fields  # is lala a declared field?
+False
 
-    >>> 'lala' in product.fields  # is lala a declared field?
-    False
 
 Setting field values
 --------------------
 
-::
+>>> product['last_updated'] = 'today'
+>>> product['last_updated']
+today
 
-    >>> product['last_updated'] = 'today'
-    >>> product['last_updated']
-    today
+>>> product['lala'] = 'test' # setting unknown field
+Traceback (most recent call last):
+    ...
+KeyError: 'Product does not support field: lala'
 
-    >>> product['lala'] = 'test' # setting unknown field
-    Traceback (most recent call last):
-        ...
-    KeyError: 'Product does not support field: lala'
 
 Accessing all populated values
 ------------------------------
 
-To access all populated values, just use the typical `dict API`_::
+To access all populated values, just use the typical `dict API`_:
 
-    >>> product.keys()
-    ['price', 'name']
+>>> product.keys()
+['price', 'name']
+
+>>> product.items()
+[('price', 1000), ('name', 'Desktop PC')]
+
+
+.. _copying-items:
+
+Copying items
+-------------
+
+To copy an item, you must first decide whether you want a shallow copy or a
+deep copy.
+
+If your item contains mutable_ values like lists or dictionaries, a shallow
+copy will keep references to the same mutable values across all different
+copies.
+
+.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
+
+For example, if you have an item with a list of tags, and you create a shallow
+copy of that item, both the original item and the copy have the same list of
+tags. Adding a tag to the list of one of the items will add the tag to the
+other item as well.
+
+If that is not the desired behavior, use a deep copy instead.
+
+See the `documentation of the copy module`_ for more information.
+
+.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
+
+To create a shallow copy of an item, you can either call
+:meth:`~scrapy.item.Item.copy` on an existing item
+(``product2 = product.copy()``) or instantiate your item class from an existing
+item (``product2 = Product(product)``).
+
+To create a deep copy, call :meth:`~scrapy.item.Item.deepcopy` instead
+(``product2 = product.deepcopy()``).
 
-    >>> product.items()
-    [('price', 1000), ('name', 'Desktop PC')]
 
 Other common tasks
 ------------------
 
-Copying items::
+Creating dicts from items:
 
-    >>> product2 = Product(product)
-    >>> print(product2)
-    Product(name='Desktop PC', price=1000)
+>>> dict(product) # create a dict from all populated values
+{'price': 1000, 'name': 'Desktop PC'}
 
-    >>> product3 = product2.copy()
-    >>> print(product3)
-    Product(name='Desktop PC', price=1000)
+Creating items from dicts:
 
-Creating dicts from items::
+>>> Product({'name': 'Laptop PC', 'price': 1500})
+Product(price=1500, name='Laptop PC')
 
-    >>> dict(product) # create a dict from all populated values
-    {'price': 1000, 'name': 'Desktop PC'}
+>>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
+Traceback (most recent call last):
+    ...
+KeyError: 'Product does not support field: lala'
 
-Creating items from dicts::
-
-    >>> Product({'name': 'Laptop PC', 'price': 1500})
-    Product(price=1500, name='Laptop PC')
-
-    >>> Product({'name': 'Laptop PC', 'lala': 1500}) # warning: unknown field in dict
-    Traceback (most recent call last):
-        ...
-    KeyError: 'Product does not support field: lala'
 
 Extending Items
 ===============
@@ -211,8 +235,12 @@ Item objects
 
     Return a new Item optionally initialized from the given argument.
 
-    Items replicate the standard `dict API`_, including its constructor. The
-    only additional attribute provided by Items is:
+    Items replicate the standard `dict API`_, including its ``__init__`` method, and
+    also provide the following additional API members:
+
+    .. automethod:: copy
+
+    .. automethod:: deepcopy
 
     .. attribute:: fields
 
@@ -237,3 +265,9 @@ Field objects
 .. _dict: https://docs.python.org/2/library/stdtypes.html#dict
 
 
+Other classes related to Item
+=============================
+
+.. autoclass:: BaseItem
+
+.. autoclass:: ItemMeta

docs/topics/jobs.rst

@@ -22,7 +22,7 @@ Job directory
 
 To enable persistence support you just need to define a *job directory* through
 the ``JOBDIR`` setting. This directory will be for storing all required data to
-keep the state of a single job (ie. a spider run).  It's important to note that
+keep the state of a single job (i.e. a spider run).  It's important to note that
 this directory must not be shared by different spiders, or even different
 jobs/runs of the same spider, as it's meant to be used for storing the state of
 a *single* job.
@@ -30,7 +30,7 @@ a *single* job.
 How to use it
 =============
 
-To start a spider with persistence supported enabled, run it like this::
+To start a spider with persistence support enabled, run it like this::
 
     scrapy crawl somespider -s JOBDIR=crawls/somespider-1
 
@@ -68,36 +68,17 @@ Cookies may expire. So, if you don't resume your spider quickly the requests
 scheduled may no longer work. This won't be an issue if you spider doesn't rely
 on cookies.
 
+
+.. _request-serialization:
+
 Request serialization
 ---------------------
 
-Requests must be serializable by the `pickle` module, in order for persistence
-to work, so you should make sure that your requests are serializable.
-
-The most common issue here is to use ``lambda`` functions on request callbacks that
-can't be persisted.
-
-So, for example, this won't work::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
-
-    def other_callback(self, response, somearg):
-        print("the argument passed is: %s" % somearg)
-
-But this will::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
-
-    def other_callback(self, response):
-        somearg = response.meta['somearg']
-        print("the argument passed is: %s" % somearg)
+For persistence to work, :class:`~scrapy.http.Request` objects must be
+serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
+values passed to their ``__init__`` method, which must be methods of the
+running :class:`~scrapy.spiders.Spider` class.
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
-
-.. _pickle: https://docs.python.org/library/pickle.html

docs/topics/leaks.rst

@@ -17,8 +17,8 @@ what is known as a "memory leak".
 
 To help debugging memory leaks, Scrapy provides a built-in mechanism for
 tracking objects references called :ref:`trackref <topics-leaks-trackrefs>`,
-and you can also use a third-party library called :ref:`Guppy
-<topics-leaks-guppy>` for more advanced memory debugging (see below for more
+and you can also use a third-party library called :ref:`muppy
+<topics-leaks-muppy>` for more advanced memory debugging (see below for more
 info). Both mechanisms must be used from the :ref:`Telnet Console
 <topics-telnetconsole>`.
 
@@ -27,10 +27,11 @@ Common causes of memory leaks
 
 It happens quite often (sometimes by accident, sometimes on purpose) that the
 Scrapy developer passes objects referenced in Requests (for example, using the
-:attr:`~scrapy.http.Request.meta` attribute or the request callback function)
-and that effectively bounds the lifetime of those referenced objects to the
-lifetime of the Request. This is, by far, the most common cause of memory leaks
-in Scrapy projects, and a quite difficult one to debug for newcomers.
+:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
+attributes or the request callback function) and that effectively bounds the
+lifetime of those referenced objects to the lifetime of the Request. This is,
+by far, the most common cause of memory leaks in Scrapy projects, and a quite
+difficult one to debug for newcomers.
 
 In big projects, the spiders are typically written by different people and some
 of those spiders could be "leaking" and thus affecting the rest of the other
@@ -48,7 +49,8 @@ Too Many Requests?
 
 By default Scrapy keeps the request queue in memory; it includes
 :class:`~scrapy.http.Request` objects and all objects
-referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
+referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
+and :attr:`~scrapy.http.Request.meta`).
 While not necessarily a leak, this can take a lot of memory. Enabling
 :ref:`persistent job queue <topics-jobs>` could help keeping memory usage
 in control.
@@ -101,14 +103,14 @@ Let's see a concrete example of a hypothetical case of memory leaks.
 Suppose we have some spider with a line similar to this one::
 
     return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
-        callback=self.parse, meta={referer: response})
+                   callback=self.parse, cb_kwargs={'referer': response})
 
 That line is passing a response reference inside a request which effectively
 ties the response lifetime to the requests' one, and that would definitely
 cause memory leaks.
 
 Let's see how we can discover the cause (without knowing it
-a-priori, of course) by using the ``trackref`` tool.
+a priori, of course) by using the ``trackref`` tool.
 
 After the crawler is running for a few minutes and we notice its memory usage
 has grown a lot, we can enter its telnet console and check the live
@@ -130,21 +132,21 @@ and check the code of the spider to discover the nasty line that is
 generating the leaks (passing response references inside requests).
 
 Sometimes extra information about live objects can be helpful.
-Let's check the oldest response::
+Let's check the oldest response:
 
-    >>> from scrapy.utils.trackref import get_oldest
-    >>> r = get_oldest('HtmlResponse')
-    >>> r.url
-    'http://www.somenastyspider.com/product.php?pid=123'
+>>> from scrapy.utils.trackref import get_oldest
+>>> r = get_oldest('HtmlResponse')
+>>> r.url
+'http://www.somenastyspider.com/product.php?pid=123'
 
 If you want to iterate over all objects, instead of getting the oldest one, you
-can use the :func:`scrapy.utils.trackref.iter_all` function::
+can use the :func:`scrapy.utils.trackref.iter_all` function:
 
-    >>> from scrapy.utils.trackref import iter_all
-    >>> [r.url for r in iter_all('HtmlResponse')]
-    ['http://www.somenastyspider.com/product.php?pid=123',
-     'http://www.somenastyspider.com/product.php?pid=584',
-    ...
+>>> from scrapy.utils.trackref import iter_all
+>>> [r.url for r in iter_all('HtmlResponse')]
+['http://www.somenastyspider.com/product.php?pid=123',
+ 'http://www.somenastyspider.com/product.php?pid=584',
+...]
 
 Too many spiders?
 -----------------
@@ -153,10 +155,10 @@ If your project has too many spiders executed in parallel,
 the output of :func:`prefs()` can be difficult to read.
 For this reason, that function has a ``ignore`` argument which can be used to
 ignore a particular class (and all its subclases). For
-example, this won't show any live references to spiders::
+example, this won't show any live references to spiders:
 
-    >>> from scrapy.spiders import Spider
-    >>> prefs(ignore=Spider)
+>>> from scrapy.spiders import Spider
+>>> prefs(ignore=Spider)
 
 .. module:: scrapy.utils.trackref
    :synopsis: Track references of live objects
@@ -168,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
 
 .. class:: object_ref
 
-    Inherit from this class (instead of object) if you want to track live
+    Inherit from this class if you want to track live
     instances with the ``trackref`` module.
 
 .. function:: print_live_refs(class_name, ignore=NoneType)
@@ -191,9 +193,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
     ``None`` if none is found. Use :func:`print_live_refs` first to get a list
     of all tracked live objects per class name.
 
-.. _topics-leaks-guppy:
+.. _topics-leaks-muppy:
 
-Debugging memory leaks with Guppy
+Debugging memory leaks with muppy
 =================================
 
 ``trackref`` provides a very convenient mechanism for tracking down memory
@@ -201,64 +203,10 @@ leaks, but it only keeps track of the objects that are more likely to cause
 memory leaks (Requests, Responses, Items, and Selectors). However, there are
 other cases where the memory leaks could come from other (more or less obscure)
 objects. If this is your case, and you can't find your leaks using ``trackref``,
-you still have another resource: the `Guppy library`_.
-If you're using Python3, see :ref:`topics-leaks-muppy`.
+you still have another resource: the muppy library.
 
-.. _Guppy library: https://pypi.python.org/pypi/guppy
 
-If you use ``pip``, you can install Guppy with the following command::
-
-    pip install guppy
-
-The telnet console also comes with a built-in shortcut (``hpy``) for accessing
-Guppy heap objects. Here's an example to view all Python objects available in
-the heap using Guppy::
-
-    >>> x = hpy.heap()
-    >>> x.bytype
-    Partition of a set of 297033 objects. Total size = 52587824 bytes.
-     Index  Count   %     Size   % Cumulative  % Type
-         0  22307   8 16423880  31  16423880  31 dict
-         1 122285  41 12441544  24  28865424  55 str
-         2  68346  23  5966696  11  34832120  66 tuple
-         3    227   0  5836528  11  40668648  77 unicode
-         4   2461   1  2222272   4  42890920  82 type
-         5  16870   6  2024400   4  44915320  85 function
-         6  13949   5  1673880   3  46589200  89 types.CodeType
-         7  13422   5  1653104   3  48242304  92 list
-         8   3735   1  1173680   2  49415984  94 _sre.SRE_Pattern
-         9   1209   0   456936   1  49872920  95 scrapy.http.headers.Headers
-    <1676 more rows. Type e.g. '_.more' to view.>
-
-You can see that most space is used by dicts. Then, if you want to see from
-which attribute those dicts are referenced, you could do::
-
-    >>> x.bytype[0].byvia
-    Partition of a set of 22307 objects. Total size = 16423880 bytes.
-     Index  Count   %     Size   % Cumulative  % Referred Via:
-         0  10982  49  9416336  57   9416336  57 '.__dict__'
-         1   1820   8  2681504  16  12097840  74 '.__dict__', '.func_globals'
-         2   3097  14  1122904   7  13220744  80
-         3    990   4   277200   2  13497944  82 "['cookies']"
-         4    987   4   276360   2  13774304  84 "['cache']"
-         5    985   4   275800   2  14050104  86 "['meta']"
-         6    897   4   251160   2  14301264  87 '[2]'
-         7      1   0   196888   1  14498152  88 "['moduleDict']", "['modules']"
-         8    672   3   188160   1  14686312  89 "['cb_kwargs']"
-         9     27   0   155016   1  14841328  90 '[1]'
-    <333 more rows. Type e.g. '_.more' to view.>
-
-As you can see, the Guppy module is very powerful but also requires some deep
-knowledge about Python internals. For more info about Guppy, refer to the
-`Guppy documentation`_.
-
-.. _Guppy documentation: http://guppy-pe.sourceforge.net/
-
-.. _topics-leaks-muppy:
-
-Debugging memory leaks with muppy
-=================================
-If you're using Python 3, you can use muppy from `Pympler`_.
+You can use muppy from `Pympler`_.
 
 .. _Pympler: https://pypi.org/project/Pympler/
 
@@ -267,32 +215,32 @@ If you use ``pip``, you can install muppy with the following command::
     pip install Pympler
 
 Here's an example to view all Python objects available in
-the heap using muppy::
+the heap using muppy:
 
-    >>> from pympler import muppy
-    >>> all_objects = muppy.get_objects()
-    >>> len(all_objects)
-    28667
-    >>> from pympler import summary
-    >>> suml = summary.summarize(all_objects)
-    >>> summary.print_(suml)
-                                   types |   # objects |   total size
-    ==================================== | =========== | ============
-                             <class 'str |        9822 |      1.10 MB
-                            <class 'dict |        1658 |    856.62 KB
-                            <class 'type |         436 |    443.60 KB
-                            <class 'code |        2974 |    419.56 KB
-              <class '_io.BufferedWriter |           2 |    256.34 KB
-                             <class 'set |         420 |    159.88 KB
-              <class '_io.BufferedReader |           1 |    128.17 KB
-              <class 'wrapper_descriptor |        1130 |     88.28 KB
-                           <class 'tuple |        1304 |     86.57 KB
-                         <class 'weakref |        1013 |     79.14 KB
-      <class 'builtin_function_or_method |         958 |     67.36 KB
-               <class 'method_descriptor |         865 |     60.82 KB
-                     <class 'abc.ABCMeta |          62 |     59.96 KB
-                            <class 'list |         446 |     58.52 KB
-                             <class 'int |        1425 |     43.20 KB
+>>> from pympler import muppy
+>>> all_objects = muppy.get_objects()
+>>> len(all_objects)
+28667
+>>> from pympler import summary
+>>> suml = summary.summarize(all_objects)
+>>> summary.print_(suml)
+                               types |   # objects |   total size
+==================================== | =========== | ============
+                         <class 'str |        9822 |      1.10 MB
+                        <class 'dict |        1658 |    856.62 KB
+                        <class 'type |         436 |    443.60 KB
+                        <class 'code |        2974 |    419.56 KB
+          <class '_io.BufferedWriter |           2 |    256.34 KB
+                         <class 'set |         420 |    159.88 KB
+          <class '_io.BufferedReader |           1 |    128.17 KB
+          <class 'wrapper_descriptor |        1130 |     88.28 KB
+                       <class 'tuple |        1304 |     86.57 KB
+                     <class 'weakref |        1013 |     79.14 KB
+  <class 'builtin_function_or_method |         958 |     67.36 KB
+           <class 'method_descriptor |         865 |     60.82 KB
+                 <class 'abc.ABCMeta |          62 |     59.96 KB
+                        <class 'list |         446 |     58.52 KB
+                         <class 'int |        1425 |     43.20 KB
 
 For more info about muppy, refer to the `muppy documentation`_.
 
@@ -309,9 +257,9 @@ though neither Scrapy nor your project are leaking memory. This is due to a
 (not so well) known problem of Python, which may not return released memory to
 the operating system in some cases. For more information on this issue see:
 
-* `Python Memory Management <http://www.evanjones.ca/python-memory.html>`_
-* `Python Memory Management Part 2 <http://www.evanjones.ca/python-memory-part2.html>`_
-* `Python Memory Management Part 3 <http://www.evanjones.ca/python-memory-part3.html>`_
+* `Python Memory Management <https://www.evanjones.ca/python-memory.html>`_
+* `Python Memory Management Part 2 <https://www.evanjones.ca/python-memory-part2.html>`_
+* `Python Memory Management Part 3 <https://www.evanjones.ca/python-memory-part3.html>`_
 
 The improvements proposed by Evan Jones, which are detailed in `this paper`_,
 got merged in Python 2.5, but this only reduces the problem, it doesn't fix it
@@ -325,7 +273,7 @@ completely. To quote the paper:
     to move to a compacting garbage collector, which is able to move objects in
     memory. This would require significant changes to the Python interpreter.*
 
-.. _this paper: http://www.evanjones.ca/memoryallocator/
+.. _this paper: https://www.evanjones.ca/memoryallocator/
 
 To keep memory consumption reasonable you can split the job into several
 smaller jobs or enable :ref:`persistent job queue <topics-jobs>`

docs/topics/link-extractors.rst

@@ -4,46 +4,33 @@
 Link Extractors
 ===============
 
-Link extractors are objects whose only purpose is to extract links from web
-pages (:class:`scrapy.http.Response` objects) which will be eventually
-followed.
+A link extractor is an object that extracts links from responses.
 
-There is ``scrapy.linkextractors.LinkExtractor`` available
-in Scrapy, but you can create your own custom Link Extractors to suit your
-needs by implementing a simple interface.
-
-The only public method that every link extractor has is ``extract_links``,
-which receives a :class:`~scrapy.http.Response` object and returns a list
-of :class:`scrapy.link.Link` objects. Link extractors are meant to be
-instantiated once and their ``extract_links`` method called several times
-with different responses to extract links to follow.
-
-Link extractors are used in the :class:`~scrapy.spiders.CrawlSpider`
-class (available in Scrapy), through a set of rules, but you can also use it in
-your spiders, even if you don't subclass from
-:class:`~scrapy.spiders.CrawlSpider`, as its purpose is very simple: to
-extract links.
+The ``__init__`` method of
+:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
+determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
+<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
+list of matching :class:`scrapy.link.Link` objects from a
+:class:`~scrapy.http.Response` object.
 
+Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
+through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
+extractors in regular spiders.
 
 .. _topics-link-extractors-ref:
 
-Built-in link extractors reference
-==================================
+Link extractor reference
+========================
 
 .. module:: scrapy.linkextractors
    :synopsis: Link extractors classes
 
-Link extractors classes bundled with Scrapy are provided in the
-:mod:`scrapy.linkextractors` module.
-
-The default link extractor is ``LinkExtractor``, which is the same as
-:class:`~.LxmlLinkExtractor`::
+The link extractor class is
+:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
+can also be imported as ``scrapy.linkextractors.LinkExtractor``::
 
     from scrapy.linkextractors import LinkExtractor
 
-There used to be other link extractor classes in previous Scrapy versions,
-but they are deprecated now.
-
 LxmlLinkExtractor
 -----------------
 
@@ -62,7 +49,7 @@ LxmlLinkExtractor
     :type allow: a regular expression (or list of)
 
     :param deny: a single regular expression (or list of regular expressions)
-        that the (absolute) urls must match in order to be excluded (ie. not
+        that the (absolute) urls must match in order to be excluded (i.e. not
         extracted). It has precedence over the ``allow`` parameter. If not
         given (or empty) it won't exclude any links.
     :type deny: a regular expression (or list of)
@@ -77,9 +64,13 @@ LxmlLinkExtractor
 
     :param deny_extensions: a single value or list of strings containing
         extensions that should be ignored when extracting links.
-        If not given, it will default to the
-        ``IGNORED_EXTENSIONS`` list defined in the
-        `scrapy.linkextractors`_ package.
+        If not given, it will default to
+        :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
+
+        .. versionchanged:: 2.0
+           :data:`~scrapy.linkextractors.IGNORED_EXTENSIONS` now includes
+           ``7z``, ``7zip``, ``apk``, ``bz2``, ``cdr``, ``dmg``, ``ico``,
+           ``iso``, ``tar``, ``tar.gz``, ``webm``, and ``xz``.
     :type deny_extensions: list
 
     :param restrict_xpaths: is an XPath (or list of XPath's) which defines
@@ -93,6 +84,12 @@ LxmlLinkExtractor
         Has the same behaviour as ``restrict_xpaths``.
     :type restrict_css: str or list
 
+    :param restrict_text: a single regular expression (or list of regular expressions)
+        that the link's text must match in order to be extracted. If not
+        given (or empty), it will match all links. If a list of regular expressions is
+        given, the link will be extracted if it matches at least one.
+    :type restrict_text: a regular expression (or list of)
+
     :param tags: a tag or a list of tags to consider when extracting links.
         Defaults to ``('a', 'area')``.
     :type tags: str or list
@@ -146,4 +143,6 @@ LxmlLinkExtractor
         from elements or attributes which allow leading/trailing whitespaces).
     :type strip: boolean
 
+    .. automethod:: extract_links
+
 .. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py

docs/topics/loaders.rst

@@ -26,7 +26,7 @@ Using Item Loaders to populate items
 
 To use an Item Loader, you must first instantiate it. You can either
 instantiate it with a dict-like object (e.g. Item or dict) or without one, in
-which case an Item is automatically instantiated in the Item Loader constructor
+which case an Item is automatically instantiated in the Item Loader ``__init__`` method
 using the Item class specified in the :attr:`ItemLoader.default_item_class`
 attribute.
 
@@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
 the same item field; the Item Loader will know how to "join" those values later
 using a proper processing function.
 
+.. note:: Collected data is internally stored as lists,
+   allowing to add several values to the same field.
+   If an ``item`` argument is passed when creating a loader,
+   each of the item's values will be stored as-is if it's already
+   an iterable, or wrapped with a list if it's a single value.
+
 Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
 the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
 chapter <topics-items>`::
@@ -128,28 +134,17 @@ So what happens is:
 It's worth noticing that processors are just callable objects, which are called
 with the data to be parsed, and return a parsed value. So you can use any
 function as input or output processor. The only requirement is that they must
-accept one (and only one) positional argument, which will be an iterator.
+accept one (and only one) positional argument, which will be an iterable.
 
-.. note:: Both input and output processors must receive an iterator as their
+.. versionchanged:: 2.0
+   Processors no longer need to be methods.
+
+.. note:: Both input and output processors must receive an iterable as their
    first argument. The output of those functions can be anything. The result of
    input processors will be appended to an internal list (in the Loader)
    containing the collected values (for that field). The result of the output
    processors is the value that will be finally assigned to the item.
 
-If you want to use a plain function as a processor, make sure it receives
-``self`` as the first argument::
-
-    def lowercase_processor(self, values):
-        for v in values:
-            yield v.lower()
-
-    class MyItemLoader(ItemLoader):
-        name_in = lowercase_processor
-
-This is because whenever a function is assigned as a class variable, it becomes
-a method and would be passed the instance as the the first argument when being
-called. See `this answer on stackoverflow`_ for more details.
-
 The other thing you need to keep in mind is that the values returned by input
 processors are collected internally (in lists) and then passed to output
 processors to populate the fields.
@@ -157,7 +152,7 @@ processors to populate the fields.
 Last, but not least, Scrapy comes with some :ref:`commonly used processors
 <topics-loaders-available-processors>` built-in for convenience.
 
-.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
+
 
 Declaring Item Loaders
 ======================
@@ -214,14 +209,12 @@ metadata. Here is an example::
             output_processor=TakeFirst(),
         )
 
-::
-
-    >>> from scrapy.loader import ItemLoader
-    >>> il = ItemLoader(item=Product())
-    >>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
-    >>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
-    >>> il.load_item()
-    {'name': u'Welcome to my website', 'price': u'1000'}
+>>> from scrapy.loader import ItemLoader
+>>> il = ItemLoader(item=Product())
+>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
+>>> il.add_value('price', [u'&euro;', u'<span>1000</span>'])
+>>> il.load_item()
+{'name': u'Welcome to my website', 'price': u'1000'}
 
 The precedence order, for both input and output processors, is as follows:
 
@@ -265,7 +258,7 @@ There are several ways to modify Item Loader context values:
       loader.context['unit'] = 'cm'
 
 2. On Item Loader instantiation (the keyword arguments of Item Loader
-   constructor are stored in the Item Loader context)::
+   ``__init__`` method are stored in the Item Loader context)::
 
       loader = ItemLoader(product, unit='cm')
 
@@ -286,7 +279,7 @@ ItemLoader objects
     given, one is instantiated automatically using the class in
     :attr:`default_item_class`.
 
-    When instantiated with a `selector` or a `response` parameters
+    When instantiated with a ``selector`` or a ``response`` parameters
     the :class:`ItemLoader` class provides convenient mechanisms for extracting
     data from web pages using :ref:`selectors <topics-selectors>`.
 
@@ -322,11 +315,11 @@ ItemLoader objects
             applied before processors
         :type re: str or compiled regex
 
-        Examples::
+        Examples:
 
-            >>> from scrapy.loader.processors import TakeFirst
-            >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
-            'FOO`
+        >>> from scrapy.loader.processors import TakeFirst
+        >>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
+        'FOO`
 
     .. method:: add_value(field_name, value, \*processors, \**kwargs)
 
@@ -485,6 +478,8 @@ ItemLoader objects
     .. attribute:: item
 
         The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
+        This is mostly used as a property so when attempting to override this
+        value, you may want to check out :attr:`default_item_class` first.
 
     .. attribute:: context
 
@@ -494,7 +489,7 @@ ItemLoader objects
     .. attribute:: default_item_class
 
         An Item class (or factory), used to instantiate items when not given in
-        the constructor.
+        the ``__init__`` method.
 
     .. attribute:: default_input_processor
 
@@ -509,15 +504,15 @@ ItemLoader objects
     .. attribute:: default_selector_class
 
         The class used to construct the :attr:`selector` of this
-        :class:`ItemLoader`, if only a response is given in the constructor.
-        If a selector is given in the constructor this attribute is ignored.
+        :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
+        If a selector is given in the ``__init__`` method this attribute is ignored.
         This attribute is sometimes overridden in subclasses.
 
     .. attribute:: selector
 
         The :class:`~scrapy.selector.Selector` object to extract data from.
-        It's either the selector given in the constructor or one created from
-        the response given in the constructor using the
+        It's either the selector given in the ``__init__`` method or one created from
+        the response given in the ``__init__`` method using the
         :attr:`default_selector_class`. This attribute is meant to be
         read-only.
 
@@ -642,46 +637,46 @@ Here is a list of all built-in processors:
 .. class:: Identity
 
     The simplest processor, which doesn't do anything. It returns the original
-    values unchanged. It doesn't receive any constructor arguments, nor does it
+    values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
     accept Loader contexts.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import Identity
-        >>> proc = Identity()
-        >>> proc(['one', 'two', 'three'])
-        ['one', 'two', 'three']
+    >>> from scrapy.loader.processors import Identity
+    >>> proc = Identity()
+    >>> proc(['one', 'two', 'three'])
+    ['one', 'two', 'three']
 
 .. class:: TakeFirst
 
     Returns the first non-null/non-empty value from the values received,
     so it's typically used as an output processor to single-valued fields.
-    It doesn't receive any constructor arguments, nor does it accept Loader contexts.
+    It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import TakeFirst
-        >>> proc = TakeFirst()
-        >>> proc(['', 'one', 'two', 'three'])
-        'one'
+    >>> from scrapy.loader.processors import TakeFirst
+    >>> proc = TakeFirst()
+    >>> proc(['', 'one', 'two', 'three'])
+    'one'
 
 .. class:: Join(separator=u' ')
 
-    Returns the values joined with the separator given in the constructor, which
+    Returns the values joined with the separator given in the ``__init__`` method, which
     defaults to ``u' '``. It doesn't accept Loader contexts.
 
     When using the default separator, this processor is equivalent to the
     function: ``u' '.join``
 
-    Examples::
+    Examples:
 
-        >>> from scrapy.loader.processors import Join
-        >>> proc = Join()
-        >>> proc(['one', 'two', 'three'])
-        'one two three'
-        >>> proc = Join('<br>')
-        >>> proc(['one', 'two', 'three'])
-        'one<br>two<br>three'
+    >>> from scrapy.loader.processors import Join
+    >>> proc = Join()
+    >>> proc(['one', 'two', 'three'])
+    'one two three'
+    >>> proc = Join('<br>')
+    >>> proc(['one', 'two', 'three'])
+    'one<br>two<br>three'
 
 .. class:: Compose(\*functions, \**default_loader_context)
 
@@ -694,18 +689,18 @@ Here is a list of all built-in processors:
     By default, stop process on ``None`` value. This behaviour can be changed by
     passing keyword argument ``stop_on_none=False``.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import Compose
-        >>> proc = Compose(lambda v: v[0], str.upper)
-        >>> proc(['hello', 'world'])
-        'HELLO'
+    >>> from scrapy.loader.processors import Compose
+    >>> proc = Compose(lambda v: v[0], str.upper)
+    >>> proc(['hello', 'world'])
+    'HELLO'
 
     Each function can optionally receive a ``loader_context`` parameter. For
     those which do, this processor will pass the currently active :ref:`Loader
     context <topics-loaders-context>` through that parameter.
 
-    The keyword arguments passed in the constructor are used as the default
+    The keyword arguments passed in the ``__init__`` method are used as the default
     Loader context values passed to each function call. However, the final
     Loader context values passed to functions are overridden with the currently
     active Loader context accessible through the :meth:`ItemLoader.context`
@@ -738,41 +733,41 @@ Here is a list of all built-in processors:
     :meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
     <topics-selectors>`, which returns a list of unicode strings.
 
-    The example below should clarify how it works::
+    The example below should clarify how it works:
 
-        >>> def filter_world(x):
-        ...     return None if x == 'world' else x
-        ...
-        >>> from scrapy.loader.processors import MapCompose
-        >>> proc = MapCompose(filter_world, str.upper)
-        >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
-        ['HELLO, 'THIS', 'IS', 'SCRAPY']
+    >>> def filter_world(x):
+    ...     return None if x == 'world' else x
+    ...
+    >>> from scrapy.loader.processors import MapCompose
+    >>> proc = MapCompose(filter_world, str.upper)
+    >>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
+    ['HELLO, 'THIS', 'IS', 'SCRAPY']
 
     As with the Compose processor, functions can receive Loader contexts, and
-    constructor keyword arguments are used as default context values. See
+    ``__init__`` method keyword arguments are used as default context values. See
     :class:`Compose` processor for more info.
 
 .. class:: SelectJmes(json_path)
 
-    Queries the value using the json path provided to the constructor and returns the output.
+    Queries the value using the json path provided to the ``__init__`` method and returns the output.
     Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
     This processor takes only one input at a time.
 
-    Example::
+    Example:
 
-        >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
-        >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
-        >>> proc({'foo': 'bar'})
-        'bar'
-        >>> proc({'foo': {'bar': 'baz'}})
-        {'bar': 'baz'}
+    >>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
+    >>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
+    >>> proc({'foo': 'bar'})
+    'bar'
+    >>> proc({'foo': {'bar': 'baz'}})
+    {'bar': 'baz'}
 
-    Working with Json::
+    Working with Json:
 
-        >>> import json
-        >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
-        >>> proc_single_json_str('{"foo": "bar"}')
-        'bar'
-        >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
-        >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
-        ['bar']
+    >>> import json
+    >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
+    >>> proc_single_json_str('{"foo": "bar"}')
+    'bar'
+    >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
+    >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
+    ['bar']

docs/topics/logging.rst

@@ -171,9 +171,9 @@ listed in `logging's logrecord attributes docs
 <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
 respectively.
 
-If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the scrapy
+If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
 component that prints the log. It is unset by default, hence logs contain the 
-scrapy component responsible for that log output.
+Scrapy component responsible for that log output.
 
 Command-line options
 --------------------
@@ -193,6 +193,18 @@ to override some of the Scrapy settings regarding logging.
     Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
         Further documentation on available handlers
 
+.. _custom-log-formats:
+
+Custom Log Formats
+------------------
+
+A custom log format can be set for different actions by extending
+:class:`~scrapy.logformatter.LogFormatter` class and making
+:setting:`LOG_FORMATTER` point to your new class.
+ 
+.. autoclass:: scrapy.logformatter.LogFormatter
+   :members:
+
 Advanced customization
 ----------------------
 
@@ -238,22 +250,23 @@ scrapy.utils.log module
 
 .. autofunction:: configure_logging
 
-    ``configure_logging`` is automatically called when using Scrapy commands,
-    but needs to be called explicitly when running custom scripts. In that
-    case, its usage is not required but it's recommended.
+    ``configure_logging`` is automatically called when using Scrapy commands
+    or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly
+    when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
+    In that case, its usage is not required but it's recommended.
 
-    If you plan on configuring the handlers yourself is still recommended you
-    call this function, passing `install_root_handler=False`. Bear in mind
-    there won't be any log output set by default in that case.
+    Another option when running custom scripts is to manually configure the logging.
+    To do this you can use `logging.basicConfig()`_ to set a basic root handler.
 
-    To get you started on manually configuring logging's output, you can use
-    `logging.basicConfig()`_ to set a basic root handler. This is an example
-    on how to redirect ``INFO`` or higher messages to a file::
+    Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
+    so it is recommended to only use `logging.basicConfig()`_ together with
+    :class:`~scrapy.crawler.CrawlerRunner`.
+
+    This is an example on how to redirect ``INFO`` or higher messages to a file::
 
         import logging
         from scrapy.utils.log import configure_logging
 
-        configure_logging(install_root_handler=False)
         logging.basicConfig(
             filename='log.txt',
             format='%(levelname)s: %(message)s',

docs/topics/media-pipeline.rst

@@ -97,7 +97,6 @@ For Files Pipeline, use::
 
     ITEM_PIPELINES = {'scrapy.pipelines.files.FilesPipeline': 1}
 
-
 .. note::
     You can also use both the Files and Images Pipeline at the same time.
 
@@ -117,12 +116,6 @@ For the Images Pipeline, set the :setting:`IMAGES_STORE` setting::
 Supported Storage
 =================
 
-File system is currently the only officially supported storage, but there are
-also support for storing files in `Amazon S3`_ and `Google Cloud Storage`_.
-
-.. _Amazon S3: https://aws.amazon.com/s3/
-.. _Google Cloud Storage: https://cloud.google.com/storage/
-
 File system storage
 -------------------
 
@@ -132,7 +125,7 @@ For example, the following image URL::
 
     http://www.example.com/image.jpg
 
-Whose `SHA1 hash` is::
+Whose ``SHA1 hash`` is::
 
     3afec3b4765f8f0a07b78f98c07b83f013567a0a
 
@@ -148,6 +141,29 @@ Where:
 * ``full`` is a sub-directory to separate full images from thumbnails (if
   used). For more info see :ref:`topics-images-thumbnails`.
 
+.. _media-pipeline-ftp:
+
+FTP server storage
+------------------
+
+.. versionadded:: 2.0
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can point to an FTP server.
+Scrapy will automatically upload the files to the server.
+
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` should be written in one of the
+following forms::
+
+    ftp://username:password@address:port/path
+    ftp://address:port/path
+    
+If ``username`` and ``password`` are not provided, they are taken from the :setting:`FTP_USER` and
+:setting:`FTP_PASSWORD` settings respectively.
+
+FTP supports two different connection modes: active or passive. Scrapy uses
+the passive connection mode by default. To use the active connection mode instead,
+set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
+
 Amazon S3 storage
 -----------------
 
@@ -171,7 +187,7 @@ policy::
 
 For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
 
-Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
+Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
 self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
 
     AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
@@ -310,7 +326,7 @@ images.
 
 .. setting:: IMAGES_THUMBS
 
-In order use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
+In order to use this feature, you must set :setting:`IMAGES_THUMBS` to a dictionary
 where the keys are the thumbnail names and the values are their dimensions.
 
 For example::
@@ -392,6 +408,36 @@ See here the methods that you can override in your custom Files Pipeline:
 
 .. class:: FilesPipeline
 
+   .. method:: file_path(self, request, response=None, info=None)
+
+      This method is called once per downloaded item. It returns the
+      download path of the file originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>` and
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+
+      You can override this method to customize the download path of each file.
+
+      For example, if file URLs end like regular paths (e.g.
+      ``https://example.com/a/b/c/foo.png``), you can use the following
+      approach to download all files into the ``files`` folder with their
+      original filenames (e.g. ``files/foo.png``)::
+
+        import os
+        from urllib.parse import urlparse
+
+        from scrapy.pipelines.files import FilesPipeline
+
+        class MyFilesPipeline(FilesPipeline):
+
+            def file_path(self, request, response=None, info=None):
+                return 'files/' + os.path.basename(urlparse(request.url).path)
+
+      By default the :meth:`file_path` method returns
+      ``full/<request URL hash>.<extension>``.
+
    .. method:: FilesPipeline.get_media_requests(item, info)
 
       As seen on the workflow, the pipeline will get the URLs of the images to
@@ -411,8 +457,9 @@ See here the methods that you can override in your custom Files Pipeline:
       * ``success`` is a boolean which is ``True`` if the image was downloaded
         successfully or ``False`` if it failed for some reason
 
-      * ``file_info_or_error`` is a dict containing the following keys (if success
-        is ``True``) or a `Twisted Failure`_ if there was a problem.
+      * ``file_info_or_error`` is a dict containing the following keys (if
+        success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
+        there was a problem.
 
         * ``url`` - the url where the file was downloaded from. This is the url of
           the request returned from the :meth:`~get_media_requests`
@@ -475,6 +522,36 @@ See here the methods that you can override in your custom Images Pipeline:
     The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
     customizing the field names and adding custom behavior for images.
 
+   .. method:: file_path(self, request, response=None, info=None)
+
+      This method is called once per downloaded item. It returns the
+      download path of the file originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>` and
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+
+      You can override this method to customize the download path of each file.
+
+      For example, if file URLs end like regular paths (e.g.
+      ``https://example.com/a/b/c/foo.png``), you can use the following
+      approach to download all files into the ``files`` folder with their
+      original filenames (e.g. ``files/foo.png``)::
+
+        import os
+        from urllib.parse import urlparse
+
+        from scrapy.pipelines.images import ImagesPipeline
+
+        class MyImagesPipeline(ImagesPipeline):
+
+            def file_path(self, request, response=None, info=None):
+                return 'files/' + os.path.basename(urlparse(request.url).path)
+
+      By default the :meth:`file_path` method returns
+      ``full/<request URL hash>.<extension>``.
+
    .. method:: ImagesPipeline.get_media_requests(item, info)
 
       Works the same way as :meth:`FilesPipeline.get_media_requests` method,
@@ -494,10 +571,12 @@ See here the methods that you can override in your custom Images Pipeline:
       By default, the :meth:`item_completed` method returns the item.
 
 
+.. _media-pipeline-example:
+
 Custom Images pipeline example
 ==============================
 
-Here is a full example of the Images Pipeline whose methods are examplified
+Here is a full example of the Images Pipeline whose methods are exemplified
 above::
 
     import scrapy
@@ -517,5 +596,12 @@ above::
             item['image_paths'] = image_paths
             return item
 
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
+
+To enable your custom media pipeline component you must add its class import path to the
+:setting:`ITEM_PIPELINES` setting, like in the following example::
+
+   ITEM_PIPELINES = {
+       'myproject.pipelines.MyImagesPipeline': 300
+   }
+
 .. _MD5 hash: https://en.wikipedia.org/wiki/MD5

docs/topics/practices.rst

@@ -34,15 +34,16 @@ Here's an example showing how to run a single spider with it.
         # Your spider definition
         ...
 
-    process = CrawlerProcess({
-        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+    process = CrawlerProcess(settings={
+        'FEED_FORMAT': 'json',
+        'FEED_URI': 'items.json'
     })
 
     process.crawl(MySpider)
     process.start() # the script will block here until the crawling is finished
 
-Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
-acquainted with its usage details.
+Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
+documentation to get acquainted with its usage details.
 
 If you are inside a Scrapy project there are some additional helpers you can
 use to import those components within the project. You can automatically import
@@ -80,7 +81,7 @@ returned by the :meth:`CrawlerRunner.crawl
 <scrapy.crawler.CrawlerRunner.crawl>` method.
 
 Here's an example of its usage, along with a callback to manually stop the
-reactor after `MySpider` has finished running.
+reactor after ``MySpider`` has finished running.
 
 ::
 
@@ -100,7 +101,7 @@ reactor after `MySpider` has finished running.
     d.addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until the crawling is finished
 
-.. seealso:: `Twisted Reactor Overview`_.
+.. seealso:: :doc:`twisted:core/howto/reactor-basics`
 
 .. _run-multiple-spiders:
 
@@ -252,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
 .. _ProxyMesh: https://proxymesh.com/
 .. _Google cache: http://www.googleguide.com/cached_pages.html
 .. _testspiders: https://github.com/scrapinghub/testspiders
-.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
 .. _Crawlera: https://scrapinghub.com/crawlera
 .. _scrapoxy: https://scrapoxy.io/

docs/topics/request-response.rst

@@ -24,13 +24,15 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
+.. autoclass:: Request
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
     a :class:`Response`.
 
     :param url: the URL of this request
+
+        If the URL is invalid, a :exc:`ValueError` exception is raised.
     :type url: string
 
     :param callback: the function that will be called with the response of this
@@ -50,7 +52,7 @@ Request objects
     :type meta: dict
 
     :param body: the request body. If a ``unicode`` is passed, then it's encoded to
-      ``str`` using the `encoding` passed (which defaults to ``utf-8``). If
+      ``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
       ``body`` is not given, an empty string is stored. Regardless of the
       type of this argument, the final value stored will be a ``str`` (never
       ``unicode`` or ``None``).
@@ -83,17 +85,21 @@ Request objects
         .. reqmeta:: dont_merge_cookies
 
         When some site returns cookies (in a response) those are stored in the
-        cookies for that domain and will be sent again in future requests. That's
-        the typical behaviour of any regular web browser. However, if, for some
-        reason, you want to avoid merging with existing cookies you can instruct
-        Scrapy to do so by setting the ``dont_merge_cookies`` key to True in the
-        :attr:`Request.meta`.
+        cookies for that domain and will be sent again in future requests.
+        That's the typical behaviour of any regular web browser.
 
-        Example of request without merging cookies::
+        To create a request that does not send stored cookies and does not
+        store received cookies, set the ``dont_merge_cookies`` key to ``True``
+        in :attr:`request.meta <scrapy.http.Request.meta>`.
 
-            request_with_cookies = Request(url="http://www.example.com",
-                                           cookies={'currency': 'USD', 'country': 'UY'},
-                                           meta={'dont_merge_cookies': True})
+        Example of a request that sends manually-defined cookies and ignores
+        cookie storage::
+
+            Request(
+                url="http://www.example.com",
+                cookies={'currency': 'USD', 'country': 'UY'},
+                meta={'dont_merge_cookies': True},
+            )
 
         For more info see :ref:`cookies-mw`.
     :type cookies: dict or list
@@ -117,20 +123,27 @@ Request objects
 
     :param errback: a function that will be called if any exception was
        raised while processing the request. This includes pages that failed
-       with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
-       as first parameter.
+       with 404 HTTP errors and such. It receives a
+       :exc:`~twisted.python.failure.Failure` as first parameter.
        For more information,
        see :ref:`topics-request-response-ref-errbacks` below.
+
+       .. versionchanged:: 2.0
+          The *callback* parameter is no longer required when the *errback*
+          parameter is specified.
     :type errback: callable
 
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
     :type flags: list
 
+    :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+    :type cb_kwargs: dict
+
     .. attribute:: Request.url
 
         A string containing the URL of this request. Keep in mind that this
         attribute contains the escaped URL, so it can differ from the URL passed in
-        the constructor.
+        the ``__init__`` method.
 
         This attribute is read-only. To change the URL of a Request use
         :meth:`replace`.
@@ -165,6 +178,17 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.meta`` attribute.
 
+    .. attribute:: Request.cb_kwargs
+
+        A dictionary that contains arbitrary metadata for this request. Its contents
+        will be passed to the Request's callback as keyword arguments. It is empty
+        for new Requests, which means by default callbacks only get a :class:`Response`
+        object as argument.
+
+        This dict is `shallow copied`_ when the request is cloned using the
+        ``copy()`` or ``replace()`` methods, and can also be accessed, in your
+        spider, from the ``response.cb_kwargs`` attribute.
+
     .. _shallow copied: https://docs.python.org/2/library/copy.html
 
     .. method:: Request.copy()
@@ -172,14 +196,16 @@ Request objects
        Return a new Request which is a copy of this Request. See also:
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
-    .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback])
+    .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
 
        Return a Request object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
-       attribute :attr:`Request.meta` is copied by default (unless a new value
-       is given in the ``meta`` argument). See also
+       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow
+       copied by default (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
+    .. automethod:: from_curl
+
 .. _topics-request-response-ref-request-callback-arguments:
 
 Passing additional data to callback functions
@@ -200,25 +226,31 @@ Example::
         self.logger.info("Visited %s", response.url)
 
 In some cases you may be interested in passing arguments to those callback
-functions so you can receive the arguments later, in the second callback. You
-can use the :attr:`Request.meta` attribute for that.
+functions so you can receive the arguments later, in the second callback.
+The following example shows how to achieve this by using the
+:attr:`Request.cb_kwargs` attribute:
 
-Here's an example of how to pass an item using this mechanism, to populate
-different fields from different pages::
+::
 
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.meta['item'] = item
+    def parse(self, response):
+        request = scrapy.Request('http://www.example.com/index.html',
+                                 callback=self.parse_page2,
+                                 cb_kwargs=dict(main_url=response.url))
+        request.cb_kwargs['foo'] = 'bar'  # add more arguments for the callback
         yield request
 
-    def parse_page2(self, response):
-        item = response.meta['item']
-        item['other_url'] = response.url
-        yield item
+    def parse_page2(self, response, main_url, foo):
+        yield dict(
+            main_url=main_url,
+            other_url=response.url,
+            foo=foo,
+        )
 
+.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
+   Prior to that, using :attr:`Request.meta` was recommended for passing
+   information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs`
+   became the preferred way for handling user information, leaving :attr:`Request.meta`
+   for communication with components like middlewares and extensions.
 
 .. _topics-request-response-ref-errbacks:
 
@@ -228,8 +260,8 @@ Using errbacks to catch exceptions in request processing
 The errback of a request is a function that will be called when an exception
 is raise while processing it.
 
-It receives a `Twisted Failure`_ instance as first parameter and can be
-used to track connection establishment timeouts, DNS errors etc.
+It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
+be used to track connection establishment timeouts, DNS errors etc.
 
 Here's an example spider logging all errors and catching some specific
 errors if needed::
@@ -299,6 +331,7 @@ Those are:
 * :reqmeta:`dont_merge_cookies`
 * :reqmeta:`cookiejar`
 * :reqmeta:`dont_cache`
+* :reqmeta:`redirect_reasons`
 * :reqmeta:`redirect_urls`
 * :reqmeta:`bindaddress`
 * :reqmeta:`dont_obey_robotstxt`
@@ -369,11 +402,11 @@ The FormRequest class extends the base :class:`Request` with functionality for
 dealing with HTML forms. It uses `lxml.html forms`_  to pre-populate form
 fields with form data from :class:`Response` objects.
 
-.. _lxml.html forms: http://lxml.de/lxmlhtml.html#forms
+.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
 
 .. class:: FormRequest(url, [formdata, ...])
 
-    The :class:`FormRequest` class adds a new argument to the constructor. The
+    The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
     remaining arguments are the same as for the :class:`Request` class and are
     not documented here.
 
@@ -446,7 +479,7 @@ fields with form data from :class:`Response` objects.
        :type dont_click: boolean
 
        The other parameters of this class method are passed directly to the
-       :class:`FormRequest` constructor.
+       :class:`FormRequest` ``__init__`` method.
 
        .. versionadded:: 0.10.3
           The ``formname`` parameter.
@@ -489,6 +522,11 @@ method for this job. Here's an example spider which uses it::
 
     import scrapy
 
+    def authentication_failed(response):
+        # TODO: Check the contents of the response and return True if it failed
+        # or False if it succeeded.
+        pass
+
     class LoginSpider(scrapy.Spider):
         name = 'example.com'
         start_urls = ['http://www.example.com/users/login.php']
@@ -501,18 +539,55 @@ method for this job. Here's an example spider which uses it::
             )
 
         def after_login(self, response):
-            # check login succeed before going on
-            if "authentication failed" in response.body:
+            if authentication_failed(response):
                 self.logger.error("Login failed")
                 return
 
             # continue scraping with authenticated session...
 
+JsonRequest
+-----------
+
+The JsonRequest class extends the base :class:`Request` class with functionality for
+dealing with JSON requests.
+
+.. class:: JsonRequest(url, [... data, dumps_kwargs])
+
+   The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
+   remaining arguments are the same as for the :class:`Request` class and are
+   not documented here.
+
+   Using the :class:`JsonRequest` will set the ``Content-Type`` header to ``application/json``
+   and ``Accept`` header to ``application/json, text/javascript, */*; q=0.01``
+
+   :param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
+      if :attr:`Request.body` argument is provided this parameter will be ignored.
+      if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
+      set to ``'POST'`` automatically.
+   :type data: JSON serializable object
+
+   :param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
+       data into JSON format.
+   :type dumps_kwargs: dict
+
+.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
+
+JsonRequest usage example
+-------------------------
+
+Sending a JSON POST request with a JSON payload::
+
+   data = {
+       'name1': 'value1',
+       'name2': 'value2',
+   }
+   yield JsonRequest(url='http://www.example.com/post/action', data=data)
+
 
 Response objects
 ================
 
-.. class:: Response(url, [status=200, headers=None, body=b'', flags=None, request=None])
+.. autoclass:: Response
 
     A :class:`Response` object represents an HTTP response, which is usually
     downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -527,8 +602,8 @@ Response objects
        (for single valued headers) or lists (for multi-valued headers).
     :type headers: dict
 
-    :param body: the response body. To access the decoded text as str (unicode
-       in Python 2) you can use ``response.text`` from an encoding-aware
+    :param body: the response body. To access the decoded text as str you can use
+       ``response.text`` from an encoding-aware
        :ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
        such as :class:`TextResponse`.
     :type body: bytes
@@ -540,7 +615,10 @@ Response objects
 
     :param request: the initial value of the :attr:`Response.request` attribute.
         This represents the :class:`Request` that generated this response.
-    :type request: :class:`Request` object
+    :type request: scrapy.http.Request
+
+    :param certificate: an object representing the server's SSL certificate.
+    :type certificate: twisted.internet.ssl.Certificate
 
     .. attribute:: Response.url
 
@@ -595,7 +673,7 @@ Response objects
     .. attribute:: Response.meta
 
         A shortcut to the :attr:`Request.meta` attribute of the
-        :attr:`Response.request` object (ie. ``self.request.meta``).
+        :attr:`Response.request` object (i.e. ``self.request.meta``).
 
         Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta`
         attribute is propagated along redirects and retries, so you will get
@@ -603,13 +681,34 @@ Response objects
 
         .. seealso:: :attr:`Request.meta` attribute
 
+    .. attribute:: Response.cb_kwargs
+
+        .. versionadded:: 2.0
+
+        A shortcut to the :attr:`Request.cb_kwargs` attribute of the
+        :attr:`Response.request` object (i.e. ``self.request.cb_kwargs``).
+
+        Unlike the :attr:`Response.request` attribute, the
+        :attr:`Response.cb_kwargs` attribute is propagated along redirects and
+        retries, so you will get the original :attr:`Request.cb_kwargs` sent
+        from your spider.
+
+        .. seealso:: :attr:`Request.cb_kwargs` attribute
+
     .. attribute:: Response.flags
 
         A list that contains flags for this response. Flags are labels used for
-        tagging Responses. For example: `'cached'`, `'redirected`', etc. And
+        tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
         they're shown on the string representation of the Response (`__str__`
         method) which is used by the engine for logging.
 
+    .. attribute:: Response.certificate
+
+        A :class:`twisted.internet.ssl.Certificate` object representing
+        the server's SSL certificate.
+        
+        Only populated for ``https`` responses, ``None`` otherwise.
+
     .. method:: Response.copy()
 
        Returns a new Response which is a copy of this Response.
@@ -632,6 +731,8 @@ Response objects
 
     .. automethod:: Response.follow
 
+    .. automethod:: Response.follow_all
+
 
 .. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
 
@@ -652,7 +753,7 @@ TextResponse objects
     :class:`Response` class, which is meant to be used only for binary data,
     such as images, sounds or any media file.
 
-    :class:`TextResponse` objects support a new constructor argument, in
+    :class:`TextResponse` objects support a new ``__init__`` method argument, in
     addition to the base :class:`Response` objects. The remaining functionality
     is the same as for the :class:`Response` class and is not documented here.
 
@@ -678,7 +779,7 @@ TextResponse objects
 
             ``unicode(response.body)`` is not a correct way to convert response
             body to unicode: you would be using the system default encoding
-            (typically `ascii`) instead of the response encoding.
+            (typically ``ascii``) instead of the response encoding.
 
 
     .. attribute:: TextResponse.encoding
@@ -686,10 +787,10 @@ TextResponse objects
        A string with the encoding of this response. The encoding is resolved by
        trying the following mechanisms, in order:
 
-       1. the encoding passed in the constructor `encoding` argument
+       1. the encoding passed in the ``__init__`` method ``encoding`` argument
 
        2. the encoding declared in the Content-Type HTTP header. If this
-          encoding is not valid (ie. unknown), it is ignored and the next
+          encoding is not valid (i.e. unknown), it is ignored and the next
           resolution mechanism is tried.
 
        3. the encoding declared in the response body. The TextResponse class
@@ -721,10 +822,12 @@ TextResponse objects
 
     .. automethod:: TextResponse.follow
 
+    .. automethod:: TextResponse.follow_all
+
     .. method:: TextResponse.body_as_unicode()
 
         The same as :attr:`text`, but available as a method. This method is
-        kept for backwards compatibility; please prefer ``response.text``.
+        kept for backward compatibility; please prefer ``response.text``.
 
 
 HtmlResponse objects
@@ -747,5 +850,4 @@ XmlResponse objects
     adds encoding auto-discovering support by looking into the XML declaration
     line.  See :attr:`TextResponse.encoding`.
 
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
 .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241

docs/topics/selectors.rst

@@ -35,12 +35,11 @@ defines selectors to associate those styles with specific HTML elements.
     in speed and parsing accuracy to lxml.
 
 .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
-.. _lxml: http://lxml.de/
+.. _lxml: https://lxml.de/
 .. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
-.. _cssselect: https://pypi.python.org/pypi/cssselect/
-.. _XPath: https://www.w3.org/TR/xpath
+.. _XPath: https://www.w3.org/TR/xpath/all/
 .. _CSS: https://www.w3.org/TR/selectors
-.. _parsel: https://parsel.readthedocs.io/
+.. _parsel: https://parsel.readthedocs.io/en/latest/
 
 Using selectors
 ===============
@@ -51,18 +50,18 @@ Constructing selectors
 .. highlight:: python
 
 Response objects expose a :class:`~scrapy.selector.Selector` instance
-on ``.selector`` attribute::
+on ``.selector`` attribute:
 
-    >>> response.selector.xpath('//span/text()').get()
-    'good'
+>>> response.selector.xpath('//span/text()').get()
+'good'
 
 Querying responses using XPath and CSS is so common that responses include two
-more shortcuts: ``response.xpath()`` and ``response.css()``::
+more shortcuts: ``response.xpath()`` and ``response.css()``:
 
-    >>> response.xpath('//span/text()').get()
-    'good'
-    >>> response.css('span::text').get()
-    'good'
+>>> response.xpath('//span/text()').get()
+'good'
+>>> response.css('span::text').get()
+'good'
 
 Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
 constructed by passing either :class:`~scrapy.http.TextResponse` object or
@@ -74,21 +73,21 @@ shortcuts. By using ``response.selector`` or one of these shortcuts
 you can also ensure the response body is parsed only once.
 
 But if required, it is possible to use ``Selector`` directly.
-Constructing from text::
+Constructing from text:
 
-    >>> from scrapy.selector import Selector
-    >>> body = '<html><body><span>good</span></body></html>'
-    >>> Selector(text=body).xpath('//span/text()').get()
-    'good'
+>>> from scrapy.selector import Selector
+>>> body = '<html><body><span>good</span></body></html>'
+>>> Selector(text=body).xpath('//span/text()').get()
+'good'
 
 Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
-:class:`~scrapy.http.TextResponse` subclasses::
+:class:`~scrapy.http.TextResponse` subclasses:
 
-    >>> from scrapy.selector import Selector
-    >>> from scrapy.http import HtmlResponse
-    >>> response = HtmlResponse(url='http://example.com', body=body)
-    >>> Selector(response=response).xpath('//span/text()').get()
-    'good'
+>>> from scrapy.selector import Selector
+>>> from scrapy.http import HtmlResponse
+>>> response = HtmlResponse(url='http://example.com', body=body)
+>>> Selector(response=response).xpath('//span/text()').get()
+'good'
 
 ``Selector`` automatically chooses the best parsing rules
 (XML vs HTML) based on input type.
@@ -96,11 +95,11 @@ Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of
 Using selectors
 ---------------
 
-To explain how to use the selectors we'll use the `Scrapy shell` (which
+To explain how to use the selectors we'll use the ``Scrapy shell`` (which
 provides interactive testing) and an example page located in the Scrapy
 documentation server:
 
-    https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+    https://docs.scrapy.org/en/latest/_static/selectors-sample1.html
 
 .. _topics-selectors-htmlcode:
 
@@ -113,7 +112,7 @@ For the sake of completeness, here's its full HTML code:
 
 First, let's open the shell::
 
-    scrapy shell https://doc.scrapy.org/en/latest/_static/selectors-sample1.html
+    scrapy shell https://docs.scrapy.org/en/latest/_static/selectors-sample1.html
 
 Then, after the shell loads, you'll have the response available as ``response``
 shell variable, and its attached selector in ``response.selector`` attribute.
@@ -123,118 +122,118 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse
 .. highlight:: python
 
 So, by looking at the :ref:`HTML code <topics-selectors-htmlcode>` of that
-page, let's construct an XPath for selecting the text inside the title tag::
+page, let's construct an XPath for selecting the text inside the title tag:
 
-    >>> response.xpath('//title/text()')
-    [<Selector xpath='//title/text()' data='Example website'>]
+>>> response.xpath('//title/text()')
+[<Selector xpath='//title/text()' data='Example website'>]
 
 To actually extract the textual data, you must call the selector ``.get()``
-or ``.getall()`` methods, as follows::
+or ``.getall()`` methods, as follows:
 
-    >>> response.xpath('//title/text()').getall()
-    ['Example website']
-    >>> response.xpath('//title/text()').get()
-    'Example website'
+>>> response.xpath('//title/text()').getall()
+['Example website']
+>>> response.xpath('//title/text()').get()
+'Example website'
 
 ``.get()`` always returns a single result; if there are several matches,
 content of a first match is returned; if there are no matches, None
 is returned. ``.getall()`` returns a list with all results.
 
 Notice that CSS selectors can select text or attribute nodes using CSS3
-pseudo-elements::
+pseudo-elements:
 
-    >>> response.css('title::text').get()
-    'Example website'
+>>> response.css('title::text').get()
+'Example website'
 
 As you can see, ``.xpath()`` and ``.css()`` methods return a
 :class:`~scrapy.selector.SelectorList` instance, which is a list of new
-selectors. This API can be used for quickly selecting nested data::
+selectors. This API can be used for quickly selecting nested data:
 
-    >>> response.css('img').xpath('@src').getall()
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> response.css('img').xpath('@src').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
 If you want to extract only the first matched element, you can call the
 selector ``.get()`` (or its alias ``.extract_first()`` commonly used in
-previous Scrapy versions)::
+previous Scrapy versions):
 
-    >>> response.xpath('//div[@id="images"]/a/text()').get()
-    'Name: My image 1 '
+>>> response.xpath('//div[@id="images"]/a/text()').get()
+'Name: My image 1 '
 
-It returns ``None`` if no element was found::
+It returns ``None`` if no element was found:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').get() is None
-    True
+>>> response.xpath('//div[@id="not-exists"]/text()').get() is None
+True
 
 A default return value can be provided as an argument, to be used instead
 of ``None``:
 
-    >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
-    'not-found'
+>>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found')
+'not-found'
 
 Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes
-using ``.attrib`` property of a :class:`~scrapy.selector.Selector`::
+using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:
 
-    >>> [img.attrib['src'] for img in response.css('img')]
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> [img.attrib['src'] for img in response.css('img')]
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
 As a shortcut, ``.attrib`` is also available on SelectorList directly;
-it returns attributes for the first matching element::
+it returns attributes for the first matching element:
 
-    >>> response.css('img').attrib['src']
-    'image1_thumb.jpg'
+>>> response.css('img').attrib['src']
+'image1_thumb.jpg'
 
 This is most useful when only a single result is expected, e.g. when selecting
-by id, or selecting unique elements on a web page::
+by id, or selecting unique elements on a web page:
 
-    >>> response.css('base').attrib['href']
-    'http://example.com/'
+>>> response.css('base').attrib['href']
+'http://example.com/'
 
-Now we're going to get the base URL and some image links::
+Now we're going to get the base URL and some image links:
 
-    >>> response.xpath('//base/@href').get()
-    'http://example.com/'
+>>> response.xpath('//base/@href').get()
+'http://example.com/'
 
-    >>> response.css('base::attr(href)').get()
-    'http://example.com/'
+>>> response.css('base::attr(href)').get()
+'http://example.com/'
 
-    >>> response.css('base').attrib['href']
-    'http://example.com/'
+>>> response.css('base').attrib['href']
+'http://example.com/'
 
-    >>> response.xpath('//a[contains(@href, "image")]/@href').getall()
-    ['image1.html',
-     'image2.html',
-     'image3.html',
-     'image4.html',
-     'image5.html']
+>>> response.xpath('//a[contains(@href, "image")]/@href').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
 
-    >>> response.css('a[href*=image]::attr(href)').getall()
-    ['image1.html',
-     'image2.html',
-     'image3.html',
-     'image4.html',
-     'image5.html']
+>>> response.css('a[href*=image]::attr(href)').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
 
-    >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> response.xpath('//a[contains(@href, "image")]/img/@src').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
-    >>> response.css('a[href*=image] img::attr(src)').getall()
-    ['image1_thumb.jpg',
-     'image2_thumb.jpg',
-     'image3_thumb.jpg',
-     'image4_thumb.jpg',
-     'image5_thumb.jpg']
+>>> response.css('a[href*=image] img::attr(src)').getall()
+['image1_thumb.jpg',
+ 'image2_thumb.jpg',
+ 'image3_thumb.jpg',
+ 'image4_thumb.jpg',
+ 'image5_thumb.jpg']
 
 .. _topics-selectors-css-extensions:
 
@@ -255,51 +254,51 @@ that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**:
     They will most probably not work with other libraries like
     `lxml`_ or `PyQuery`_.
 
-.. _PyQuery: https://pypi.python.org/pypi/pyquery
+.. _PyQuery: https://pypi.org/project/pyquery/
 
 Examples:
 
-* ``title::text`` selects children text nodes of a descendant ``<title>`` element::
+* ``title::text`` selects children text nodes of a descendant ``<title>`` element:
 
-    >>> response.css('title::text').get()
-    'Example website'
+>>> response.css('title::text').get()
+'Example website'
 
-* ``*::text`` selects all descendant text nodes of the current selector context::
+* ``*::text`` selects all descendant text nodes of the current selector context:
 
-    >>> response.css('#images *::text').getall()
-    ['\n   ',
-     'Name: My image 1 ',
-     '\n   ',
-     'Name: My image 2 ',
-     '\n   ',
-     'Name: My image 3 ',
-     '\n   ',
-     'Name: My image 4 ',
-     '\n   ',
-     'Name: My image 5 ',
-     '\n  ']
+>>> response.css('#images *::text').getall()
+['\n   ',
+ 'Name: My image 1 ',
+ '\n   ',
+ 'Name: My image 2 ',
+ '\n   ',
+ 'Name: My image 3 ',
+ '\n   ',
+ 'Name: My image 4 ',
+ '\n   ',
+ 'Name: My image 5 ',
+ '\n  ']
 
 * ``foo::text`` returns no results if ``foo`` element exists, but contains
-  no text (i.e. text is empty)::
+  no text (i.e. text is empty):
 
-    >>> response.css('img::text').getall()
-    []
+>>> response.css('img::text').getall()
+[]
 
   This means ``.css('foo::text').get()`` could return None even if an element
-  exists. Use ``default=''`` if you always want a string::
+  exists. Use ``default=''`` if you always want a string:
 
-    >>> response.css('img::text').get()
-    >>> response.css('img::text').get(default='')
-    ''
+>>> response.css('img::text').get()
+>>> response.css('img::text').get(default='')
+''
 
-* ``a::attr(href)`` selects the *href* attribute value of descendant links::
+* ``a::attr(href)`` selects the *href* attribute value of descendant links:
 
-    >>> response.css('a::attr(href)').getall()
-    ['image1.html',
-     'image2.html',
-     'image3.html',
-     'image4.html',
-     'image5.html']
+>>> response.css('a::attr(href)').getall()
+['image1.html',
+ 'image2.html',
+ 'image3.html',
+ 'image4.html',
+ 'image5.html']
 
 .. note::
     See also: :ref:`selecting-attributes`.
@@ -309,7 +308,7 @@ Examples:
     make much sense: text nodes do not have attributes, and attribute values
     are string values already and do not have children nodes.
 
-.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors
+.. _CSS Selectors: https://www.w3.org/TR/selectors-3/#selectors
 
 .. _topics-selectors-nesting-selectors:
 
@@ -318,25 +317,24 @@ Nesting selectors
 
 The selection methods (``.xpath()`` or ``.css()``) return a list of selectors
 of the same type, so you can call the selection methods for those selectors
-too. Here's an example::
+too. Here's an example:
 
-    >>> links = response.xpath('//a[contains(@href, "image")]')
-    >>> links.getall()
-    ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
-     '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
-     '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
-     '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
-     '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
+>>> links = response.xpath('//a[contains(@href, "image")]')
+>>> links.getall()
+['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>',
+ '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>',
+ '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>',
+ '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>',
+ '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>']
 
-    >>> for index, link in enumerate(links):
-    ...     args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
-    ...     print('Link number %d points to url %r and image %r' % args)
-
-    Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
-    Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
-    Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
-    Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
-    Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
+>>> for index, link in enumerate(links):
+...     args = (index, link.xpath('@href').get(), link.xpath('img/@src').get())
+...     print('Link number %d points to url %r and image %r' % args)
+Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg'
+Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg'
+Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg'
+Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg'
+Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg'
 
 .. _selecting-attributes:
 
@@ -344,42 +342,42 @@ Selecting element attributes
 ----------------------------
 
 There are several ways to get a value of an attribute. First, one can use
-XPath syntax::
+XPath syntax:
 
-    >>> response.xpath("//a/@href").getall()
-    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+>>> response.xpath("//a/@href").getall()
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
 XPath syntax has a few advantages: it is a standard XPath feature, and
 ``@attributes`` can be used in other parts of an XPath expression - e.g.
 it is possible to filter by attribute value.
 
 Scrapy also provides an extension to CSS selectors (``::attr(...)``)
-which allows to get attribute values::
+which allows to get attribute values:
 
-    >>> response.css('a::attr(href)').getall()
-    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+>>> response.css('a::attr(href)').getall()
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
 In addition to that, there is a ``.attrib`` property of Selector.
 You can use it if you prefer to lookup attributes in Python
-code, without using XPaths or CSS extensions::
+code, without using XPaths or CSS extensions:
 
-    >>> [a.attrib['href'] for a in response.css('a')]
-    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+>>> [a.attrib['href'] for a in response.css('a')]
+['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
 This property is also available on SelectorList; it returns a dictionary
 with attributes of a first matching element. It is convenient to use when
 a selector is expected to give a single result (e.g. when selecting by element
-ID, or when selecting an unique element on a page)::
+ID, or when selecting an unique element on a page):
 
-    >>> response.css('base').attrib
-    {'href': 'http://example.com/'}
-    >>> response.css('base').attrib['href']
-    'http://example.com/'
+>>> response.css('base').attrib
+{'href': 'http://example.com/'}
+>>> response.css('base').attrib['href']
+'http://example.com/'
 
-``.attrib`` property of an empty SelectorList is empty::
+``.attrib`` property of an empty SelectorList is empty:
 
-    >>> response.css('foo').attrib
-    {}
+>>> response.css('foo').attrib
+{}
 
 Using selectors with regular expressions
 ----------------------------------------
@@ -390,21 +388,21 @@ data using regular expressions. However, unlike using ``.xpath()`` or
 can't construct nested ``.re()`` calls.
 
 Here's an example used to extract image names from the :ref:`HTML code
-<topics-selectors-htmlcode>` above::
+<topics-selectors-htmlcode>` above:
 
-    >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
-    ['My image 1',
-     'My image 2',
-     'My image 3',
-     'My image 4',
-     'My image 5']
+>>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)')
+['My image 1',
+ 'My image 2',
+ 'My image 3',
+ 'My image 4',
+ 'My image 5']
 
 There's an additional helper reciprocating ``.get()`` (and its
 alias ``.extract_first()``) for ``.re()``, named ``.re_first()``.
-Use it to extract just the first matching string::
+Use it to extract just the first matching string:
 
-    >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
-    'My image 1'
+>>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)')
+'My image 1'
 
 .. _old-extraction-api:
 
@@ -422,28 +420,28 @@ and readable code.
 
 The following examples show how these methods map to each other.
 
-1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``::
+1.  ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:
 
-     >>> response.css('a::attr(href)').get()
-     'image1.html'
-     >>> response.css('a::attr(href)').extract_first()
-     'image1.html'
+    >>> response.css('a::attr(href)').get()
+    'image1.html'
+    >>> response.css('a::attr(href)').extract_first()
+    'image1.html'
 
-2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``::
+2.  ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:
 
-     >>> response.css('a::attr(href)').getall()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
-     >>> response.css('a::attr(href)').extract()
-     ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+    >>> response.css('a::attr(href)').getall()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
+    >>> response.css('a::attr(href)').extract()
+    ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html']
 
-2. ``Selector.get()`` is the same as ``Selector.extract()``::
+3.  ``Selector.get()`` is the same as ``Selector.extract()``:
 
-     >>> response.css('a::attr(href)')[0].get()
-     'image1.html'
-     >>> response.css('a::attr(href)')[0].extract()
-     'image1.html'
+    >>> response.css('a::attr(href)')[0].get()
+    'image1.html'
+    >>> response.css('a::attr(href)')[0].extract()
+    'image1.html'
 
-4. For consistency, there is also ``Selector.getall()``, which returns a list::
+4.  For consistency, there is also ``Selector.getall()``, which returns a list:
 
     >>> response.css('a::attr(href)')[0].getall()
     ['image1.html']
@@ -481,31 +479,31 @@ with ``/``, that XPath will be absolute to the document and not relative to the
 ``Selector`` you're calling it from.
 
 For example, suppose you want to extract all ``<p>`` elements inside ``<div>``
-elements. First, you would get all ``<div>`` elements::
+elements. First, you would get all ``<div>`` elements:
 
-    >>> divs = response.xpath('//div')
+>>> divs = response.xpath('//div')
 
 At first, you may be tempted to use the following approach, which is wrong, as
 it actually extracts all ``<p>`` elements from the document, not only those
-inside ``<div>`` elements::
+inside ``<div>`` elements:
 
-    >>> for p in divs.xpath('//p'):  # this is wrong - gets all <p> from the whole document
-    ...     print(p.get())
+>>> for p in divs.xpath('//p'):  # this is wrong - gets all <p> from the whole document
+...     print(p.get())
 
-This is the proper way to do it (note the dot prefixing the ``.//p`` XPath)::
+This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):
 
-    >>> for p in divs.xpath('.//p'):  # extracts all <p> inside
-    ...     print(p.get())
+>>> for p in divs.xpath('.//p'):  # extracts all <p> inside
+...     print(p.get())
 
-Another common case would be to extract all direct ``<p>`` children::
+Another common case would be to extract all direct ``<p>`` children:
 
-    >>> for p in divs.xpath('p'):
-    ...     print(p.get())
+>>> for p in divs.xpath('p'):
+...     print(p.get())
 
 For more details about relative XPaths see the `Location Paths`_ section in the
 XPath specification.
 
-.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
+.. _Location Paths: https://www.w3.org/TR/xpath/all/#location-paths
 
 When querying by class, consider using CSS
 ------------------------------------------
@@ -521,12 +519,12 @@ for that you may end up with more elements that you want, if they have a differe
 class name that shares the string ``someclass``.
 
 As it turns out, Scrapy selectors allow you to chain selectors, so most of the time
-you can just select by class using CSS and then switch to XPath when needed::
+you can just select by class using CSS and then switch to XPath when needed:
 
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
-    >>> sel.css('.shout').xpath('./time/@datetime').getall()
-    ['2014-07-23 19:00']
+>>> from scrapy import Selector
+>>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>')
+>>> sel.css('.shout').xpath('./time/@datetime').getall()
+['2014-07-23 19:00']
 
 This is cleaner than using the verbose XPath trick shown above. Just remember
 to use the ``.`` in the XPath expressions that will follow.
@@ -538,41 +536,41 @@ Beware of the difference between //node[1] and (//node)[1]
 
 ``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them.
 
-Example::
+Example:
 
-    >>> from scrapy import Selector
-    >>> sel = Selector(text="""
-    ....:     <ul class="list">
-    ....:         <li>1</li>
-    ....:         <li>2</li>
-    ....:         <li>3</li>
-    ....:     </ul>
-    ....:     <ul class="list">
-    ....:         <li>4</li>
-    ....:         <li>5</li>
-    ....:         <li>6</li>
-    ....:     </ul>""")
-    >>> xp = lambda x: sel.xpath(x).getall()
+>>> from scrapy import Selector
+>>> sel = Selector(text="""
+....:     <ul class="list">
+....:         <li>1</li>
+....:         <li>2</li>
+....:         <li>3</li>
+....:     </ul>
+....:     <ul class="list">
+....:         <li>4</li>
+....:         <li>5</li>
+....:         <li>6</li>
+....:     </ul>""")
+>>> xp = lambda x: sel.xpath(x).getall()
 
-This gets all first ``<li>``  elements under whatever it is its parent::
+This gets all first ``<li>``  elements under whatever it is its parent:
 
-    >>> xp("//li[1]")
-    ['<li>1</li>', '<li>4</li>']
+>>> xp("//li[1]")
+['<li>1</li>', '<li>4</li>']
 
-And this gets the first ``<li>``  element in the whole document::
+And this gets the first ``<li>``  element in the whole document:
 
-    >>> xp("(//li)[1]")
-    ['<li>1</li>']
+>>> xp("(//li)[1]")
+['<li>1</li>']
 
-This gets all first ``<li>``  elements under an ``<ul>``  parent::
+This gets all first ``<li>``  elements under an ``<ul>``  parent:
 
-    >>> xp("//ul/li[1]")
-    ['<li>1</li>', '<li>4</li>']
+>>> xp("//ul/li[1]")
+['<li>1</li>', '<li>4</li>']
 
-And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document::
+And this gets the first ``<li>``  element under an ``<ul>``  parent in the whole document:
 
-    >>> xp("(//ul/li)[1]")
-    ['<li>1</li>']
+>>> xp("(//ul/li)[1]")
+['<li>1</li>']
 
 Using text nodes in a condition
 -------------------------------
@@ -584,36 +582,36 @@ This is because the expression ``.//text()`` yields a collection of text element
 And when a node-set is converted to a string, which happens when it is passed as argument to
 a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only.
 
-Example::
+Example:
 
-    >>> from scrapy import Selector
-    >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
+>>> from scrapy import Selector
+>>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>')
 
-Converting a *node-set* to string::
+Converting a *node-set* to string:
 
-    >>> sel.xpath('//a//text()').getall() # take a peek at the node-set
-    ['Click here to go to the ', 'Next Page']
-    >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
-    ['Click here to go to the ']
+>>> sel.xpath('//a//text()').getall() # take a peek at the node-set
+['Click here to go to the ', 'Next Page']
+>>> sel.xpath("string(//a[1]//text())").getall() # convert it to string
+['Click here to go to the ']
 
-A *node* converted to a string, however, puts together the text of itself plus of all its descendants::
+A *node* converted to a string, however, puts together the text of itself plus of all its descendants:
 
-    >>> sel.xpath("//a[1]").getall() # select the first node
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
-    >>> sel.xpath("string(//a[1])").getall() # convert it to string
-    ['Click here to go to the Next Page']
+>>> sel.xpath("//a[1]").getall() # select the first node
+['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+>>> sel.xpath("string(//a[1])").getall() # convert it to string
+['Click here to go to the Next Page']
 
-So, using the ``.//text()`` node-set won't select anything in this case::
+So, using the ``.//text()`` node-set won't select anything in this case:
 
-    >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
-    []
+>>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall()
+[]
 
-But using the ``.`` to mean the node, works::
+But using the ``.`` to mean the node, works:
 
-    >>> sel.xpath("//a[contains(., 'Next Page')]").getall()
-    ['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
+>>> sel.xpath("//a[contains(., 'Next Page')]").getall()
+['<a href="#">Click here to go to the <strong>Next Page</strong></a>']
 
-.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions
+.. _`XPath string function`: https://www.w3.org/TR/xpath/all/#section-String-Functions
 
 .. _topics-selectors-xpath-variables:
 
@@ -627,17 +625,17 @@ some arguments in your queries with placeholders like ``?``,
 which are then substituted with values passed with the query.
 
 Here's an example to match an element based on its "id" attribute value,
-without hard-coding it (that was shown previously)::
+without hard-coding it (that was shown previously):
 
-    >>> # `$val` used in the expression, a `val` argument needs to be passed
-    >>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
-    'Name: My image 1 '
+>>> # `$val` used in the expression, a `val` argument needs to be passed
+>>> response.xpath('//div[@id=$val]/a/text()', val='images').get()
+'Name: My image 1 '
 
 Here's another example, to find the "id" attribute of a ``<div>`` tag containing
-five ``<a>`` children (here we pass the value ``5`` as an integer)::
+five ``<a>`` children (here we pass the value ``5`` as an integer):
 
-    >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
-    'images'
+>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get()
+'images'
 
 All variable references must have a binding value when calling ``.xpath()``
 (otherwise you'll get a ``ValueError: XPath error:`` exception).
@@ -687,19 +685,19 @@ You can see several namespace declarations including a default
 .. highlight:: python
 
 Once in the shell we can try selecting all ``<link>`` objects and see that it
-doesn't work (because the Atom XML namespace is obfuscating those nodes)::
+doesn't work (because the Atom XML namespace is obfuscating those nodes):
 
-    >>> response.xpath("//link")
-    []
+>>> response.xpath("//link")
+[]
 
 But once we call the :meth:`Selector.remove_namespaces` method, all
-nodes can be accessed directly by their names::
+nodes can be accessed directly by their names:
 
-    >>> response.selector.remove_namespaces()
-    >>> response.xpath("//link")
-    [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
-     <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
-     ...
+>>> response.selector.remove_namespaces()
+>>> response.xpath("//link")
+[<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>,
+    <Selector xpath='//link' data='<link rel="next" type="application/atom+'>,
+    ...
 
 If you wonder why the namespace removal procedure isn't always called by default
 instead of having to call it manually, this is because of two reasons, which, in order
@@ -734,26 +732,25 @@ Regular expressions
 The ``test()`` function, for example, can prove quite useful when XPath's
 ``starts-with()`` or ``contains()`` are not sufficient.
 
-Example selecting links in list item with a "class" attribute ending with a digit::
+Example selecting links in list item with a "class" attribute ending with a digit:
 
-    >>> from scrapy import Selector
-    >>> doc = u"""
-    ... <div>
-    ...     <ul>
-    ...         <li class="item-0"><a href="link1.html">first item</a></li>
-    ...         <li class="item-1"><a href="link2.html">second item</a></li>
-    ...         <li class="item-inactive"><a href="link3.html">third item</a></li>
-    ...         <li class="item-1"><a href="link4.html">fourth item</a></li>
-    ...         <li class="item-0"><a href="link5.html">fifth item</a></li>
-    ...     </ul>
-    ... </div>
-    ... """
-    >>> sel = Selector(text=doc, type="html")
-    >>> sel.xpath('//li//@href').getall()
-    ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
-    >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
-    ['link1.html', 'link2.html', 'link4.html', 'link5.html']
-    >>>
+>>> from scrapy import Selector
+>>> doc = u"""
+... <div>
+...     <ul>
+...         <li class="item-0"><a href="link1.html">first item</a></li>
+...         <li class="item-1"><a href="link2.html">second item</a></li>
+...         <li class="item-inactive"><a href="link3.html">third item</a></li>
+...         <li class="item-1"><a href="link4.html">fourth item</a></li>
+...         <li class="item-0"><a href="link5.html">fifth item</a></li>
+...     </ul>
+... </div>
+... """
+>>> sel = Selector(text=doc, type="html")
+>>> sel.xpath('//li//@href').getall()
+['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html']
+>>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall()
+['link1.html', 'link2.html', 'link4.html', 'link5.html']
 
 .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular
     expressions so `lxml`_'s implementation uses hooks to Python's ``re`` module.
@@ -766,7 +763,7 @@ Set operations
 These can be handy for excluding parts of a document tree before
 extracting text elements for example.
 
-Example extracting microdata (sample content taken from http://schema.org/Product)
+Example extracting microdata (sample content taken from https://schema.org/Product)
 with groups of itemscopes and corresponding itemprops::
 
     >>> doc = u"""
@@ -849,7 +846,6 @@ with groups of itemscopes and corresponding itemprops::
     current scope: ['http://schema.org/Rating']
         properties: ['worstRating', 'ratingValue', 'bestRating']
 
-    >>>
 
 Here we first iterate over ``itemscope`` elements, and for each one,
 we look for all ``itemprops`` elements and exclude those that are themselves
@@ -877,15 +873,15 @@ For the following HTML::
 
 .. highlight:: python
 
-You can use it like this::
+You can use it like this:
 
-    >>> response.xpath('//p[has-class("foo")]')
-    [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
-     <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
-    >>> response.xpath('//p[has-class("foo", "bar-baz")]')
-    [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
-    >>> response.xpath('//p[has-class("foo", "bar")]')
-    []
+>>> response.xpath('//p[has-class("foo")]')
+[<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>,
+ <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>]
+>>> response.xpath('//p[has-class("foo", "bar-baz")]')
+[<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>]
+>>> response.xpath('//p[has-class("foo", "bar")]')
+[]
 
 So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS
 ``p.foo.bar-baz``.  Please note, that it is slower in most of the cases,
@@ -989,7 +985,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
       sel = Selector(html_response)
 
 1. Select all ``<h1>`` elements from an HTML response body, returning a list of
-   :class:`Selector` objects (ie. a :class:`SelectorList` object)::
+   :class:`Selector` objects (i.e. a :class:`SelectorList` object)::
 
       sel.xpath("//h1")
 
@@ -1016,7 +1012,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object::
       sel = Selector(xml_response)
 
 1. Select all ``<product>`` elements from an XML response body, returning a list
-   of :class:`Selector` objects (ie. a :class:`SelectorList` object)::
+   of :class:`Selector` objects (i.e. a :class:`SelectorList` object)::
 
       sel.xpath("//product")
 

docs/topics/settings.rst

@@ -30,6 +30,8 @@ Python `import search path`_.
 
 .. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
 
+.. _populating-settings:
+
 Populating the settings
 =======================
 
@@ -122,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings`
 attribute of the Crawler that is passed to ``from_crawler`` method in
 extensions, middlewares and item pipelines::
 
-    class MyExtension(object):
+    class MyExtension:
         def __init__(self, log_is_enabled=False):
             if log_is_enabled:
                 print("log is enabled!")
@@ -178,35 +180,45 @@ Default: ``None``
 The AWS secret key used by code that requires access to `Amazon Web services`_,
 such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 
-.. setting:: BOT_NAME
+.. setting:: AWS_ENDPOINT_URL
 
 AWS_ENDPOINT_URL
 ----------------
 
 Default: ``None``
 
-Endpoint URL used for S3-like self-hosted storage. Storage like Minio or s3.scality.
+Endpoint URL used for S3-like storage, for example Minio or s3.scality.
 
-.. setting:: AWS_ENDPOINT_URL
+.. setting:: AWS_USE_SSL
 
 AWS_USE_SSL
 -----------
 
 Default: ``None``
 
-Use this option if you want to disable SSL connection for communication with S3 or S3-like storage.
-By default SSL will be used.
+Use this option if you want to disable SSL connection for communication with
+S3 or S3-like storage. By default SSL will be used.
 
-.. setting:: AWS_USE_SSL
+.. setting:: AWS_VERIFY
 
 AWS_VERIFY
 ----------
 
 Default: ``None``
 
-Verify SSL connection between Scrapy and S3 or S3-like storage. By default SSL verification will occur.
+Verify SSL connection between Scrapy and S3 or S3-like storage. By default
+SSL verification will occur.
 
-.. setting:: AWS_VERIFY
+.. setting:: AWS_REGION_NAME
+
+AWS_REGION_NAME
+---------------
+
+Default: ``None``
+
+The name of the region associated with the AWS client.
+
+.. setting:: BOT_NAME
 
 BOT_NAME
 --------
@@ -214,8 +226,7 @@ BOT_NAME
 Default: ``'scrapybot'``
 
 The name of the bot implemented by this Scrapy project (also known as the
-project name). This will be used to construct the User-Agent by default, and
-also for logging.
+project name). This name will be used for the logging too.
 
 It's automatically populated with your project name when you create your
 project with the :command:`startproject` command.
@@ -237,7 +248,7 @@ CONCURRENT_REQUESTS
 
 Default: ``16``
 
-The maximum number of concurrent (ie. simultaneous) requests that will be
+The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed by the Scrapy downloader.
 
 .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
@@ -247,7 +258,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN
 
 Default: ``8``
 
-The maximum number of concurrent (ie. simultaneous) requests that will be
+The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single domain.
 
 See also: :ref:`topics-autothrottle` and its
@@ -261,7 +272,7 @@ CONCURRENT_REQUESTS_PER_IP
 
 Default: ``0``
 
-The maximum number of concurrent (ie. simultaneous) requests that will be
+The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single IP. If non-zero, the
 :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is
 used instead. In other words, concurrency limits will be applied per IP, not
@@ -318,16 +329,16 @@ Default: ``0``
 
 Scope: ``scrapy.spidermiddlewares.depth.DepthMiddleware``
 
-An integer that is used to adjust the request priority based on its depth:
+An integer that is used to adjust the :attr:`~scrapy.http.Request.priority` of
+a :class:`~scrapy.http.Request` based on its depth.
 
-- if zero (default), no priority adjustment is made from depth
-- **a positive value will decrease the priority, i.e. higher depth
-  requests will be processed later** ; this is commonly used when doing
-  breadth-first crawls (BFO)
-- a negative value will increase priority, i.e., higher depth requests
-  will be processed sooner (DFO)
+The priority of a request is adjusted as follows::
 
-See also: :ref:`faq-bfo-dfo` about tuning Scrapy for BFO or DFO.
+    request.priority = request.priority - ( depth * DEPTH_PRIORITY )
+
+As depth increases, positive values of ``DEPTH_PRIORITY`` decrease request
+priority (BFO), while negative values increase request priority (DFO). See
+also :ref:`faq-bfo-dfo`.
 
 .. note::
 
@@ -365,6 +376,21 @@ Default: ``10000``
 
 DNS in-memory cache size.
 
+.. setting:: DNS_RESOLVER
+
+DNS_RESOLVER
+------------
+
+.. versionadded:: 2.0
+
+Default: ``'scrapy.resolver.CachingThreadedResolver'``
+
+The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver``
+supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting,
+but works only with IPv4 addresses. Scrapy provides an alternative resolver,
+``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
+take the :setting:`DNS_TIMEOUT` setting into account.
+
 .. setting:: DNS_TIMEOUT
 
 DNS_TIMEOUT
@@ -425,9 +451,29 @@ or even enable client-side authentication (and various other things).
     which uses the platform's certificates to validate remote endpoints.
     **This is only available if you use Twisted>=14.0.**
 
-If you do use a custom ContextFactory, make sure it accepts a ``method``
-parameter at init (this is the ``OpenSSL.SSL`` method mapping
-:setting:`DOWNLOADER_CLIENT_TLS_METHOD`).
+If you do use a custom ContextFactory, make sure its ``__init__`` method
+accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`), a ``tls_verbose_logging``
+parameter (``bool``) and a ``tls_ciphers`` parameter (see
+:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`).
+
+.. setting:: DOWNLOADER_CLIENT_TLS_CIPHERS
+
+DOWNLOADER_CLIENT_TLS_CIPHERS
+-----------------------------
+
+Default: ``'DEFAULT'``
+
+Use  this setting to customize the TLS/SSL ciphers used by the default
+HTTP/1.1 downloader.
+
+The setting should contain a string in the `OpenSSL cipher list format`_,
+these ciphers will be used as client ciphers. Changing this setting may be
+necessary to access certain HTTPS websites: for example, you may need to use
+``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
+specific cipher that is not included in ``DEFAULT`` if a website requires it.
+
+.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 
@@ -455,6 +501,20 @@ This setting must be one of these string values:
     We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
     or above (Twisted>=14.0 if you can).
 
+.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+-------------------------------------
+
+Default: ``False``
+
+Setting this to ``True`` will enable DEBUG level messages about TLS connection
+parameters after establishing HTTPS connections. The kind of information logged
+depends on the versions of OpenSSL and pyOpenSSL.
+
+This setting is only used for the default
+:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
+
 .. setting:: DOWNLOADER_MIDDLEWARES
 
 DOWNLOADER_MIDDLEWARES
@@ -525,6 +585,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti
 When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
 per ip address instead of per domain.
 
+.. _spider-download_delay-attribute:
+
 You can also change this setting per spider by setting ``download_delay``
 spider attribute.
 
@@ -586,7 +648,7 @@ The amount of time (in secs) that the downloader will wait before timing out.
 DOWNLOAD_MAXSIZE
 ----------------
 
-Default: `1073741824` (1024MB)
+Default: ``1073741824`` (1024MB)
 
 The maximum response size (in bytes) that downloader will download.
 
@@ -607,7 +669,7 @@ If you want to disable it set to 0.
 DOWNLOAD_WARNSIZE
 -----------------
 
-Default: `33554432` (32MB)
+Default: ``33554432`` (32MB)
 
 The response size (in bytes) that downloader will start to warn.
 
@@ -745,6 +807,7 @@ Default: ``True``
 
 Whether or not to use passive mode when initiating FTP transfers.
 
+.. reqmeta:: ftp_password
 .. setting:: FTP_PASSWORD
 
 FTP_PASSWORD
@@ -763,6 +826,7 @@ in ``Request`` meta.
 
 .. _RFC 1635: https://tools.ietf.org/html/rfc1635
 
+.. reqmeta:: ftp_user
 .. setting:: FTP_USER
 
 FTP_USER
@@ -835,7 +899,7 @@ LOG_FORMAT
 
 Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
 
-String for formatting log messsages. Refer to the `Python logging documentation`_ for the whole list of available
+String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
 placeholders.
 
 .. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
@@ -853,6 +917,15 @@ directives.
 
 .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
 
+.. setting:: LOG_FORMATTER
+
+LOG_FORMATTER
+-------------
+
+Default: :class:`scrapy.logformatter.LogFormatter`
+
+The class to use for :ref:`formatting log messages <custom-log-formats>` for different actions.
+
 .. setting:: LOG_LEVEL
 
 LOG_LEVEL
@@ -884,6 +957,16 @@ Default: ``False``
 If ``True``, the logs will just contain the root path. If it is set to ``False``
 then it displays the component responsible for the log output
 
+.. setting:: LOGSTATS_INTERVAL
+
+LOGSTATS_INTERVAL
+-----------------
+
+Default: ``60.0``
+
+The interval (in seconds) between each logging printout of the stats
+by :class:`~scrapy.extensions.logstats.LogStats`.
+
 .. setting:: MEMDEBUG_ENABLED
 
 MEMDEBUG_ENABLED
@@ -1090,6 +1173,28 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
     this option is enabled by default in settings.py file generated
     by ``scrapy startproject`` command.
 
+.. setting:: ROBOTSTXT_PARSER
+
+ROBOTSTXT_PARSER
+----------------
+
+Default: ``'scrapy.robotstxt.ProtegoRobotParser'``
+
+The parser backend to use for parsing ``robots.txt`` files. For more information see
+:ref:`topics-dlmw-robots`.
+
+.. setting:: ROBOTSTXT_USER_AGENT
+
+ROBOTSTXT_USER_AGENT
+^^^^^^^^^^^^^^^^^^^^
+
+Default: ``None``
+
+The user agent string to use for matching in the robots.txt file. If ``None``,
+the User-Agent header you are sending with the request or the
+:setting:`USER_AGENT` setting (in that order) will be used for determining
+the user agent to use in the robots.txt file.
+
 .. setting:: SCHEDULER
 
 SCHEDULER
@@ -1142,9 +1247,28 @@ Type of in-memory queue used by scheduler. Other available type is:
 
 SCHEDULER_PRIORITY_QUEUE
 ------------------------
-Default: ``'queuelib.PriorityQueue'``
+Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
 
-Type of priority queue used by scheduler.
+Type of priority queue used by the scheduler. Another available type is
+``scrapy.pqueues.DownloaderAwarePriorityQueue``.
+``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
+``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
+domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue``
+does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
+
+.. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE
+
+SCRAPER_SLOT_MAX_ACTIVE_SIZE
+----------------------------
+
+.. versionadded:: 2.0
+
+Default: ``5_000_000``
+
+Soft limit (in bytes) for response data being processed.
+
+While the sum of the sizes of all responses being processed is above this value,
+Scrapy does not process new requests.
 
 .. setting:: SPIDER_CONTRACTS
 
@@ -1169,7 +1293,7 @@ Default::
         'scrapy.contracts.default.ScrapesContract': 3,
     }
 
-A dict containing the scrapy contracts enabled by default in Scrapy. You should
+A dict containing the Scrapy contracts enabled by default in Scrapy. You should
 never modify this setting in your project, modify :setting:`SPIDER_CONTRACTS`
 instead. For more info see :ref:`topics-contracts`.
 
@@ -1200,7 +1324,7 @@ SPIDER_LOADER_WARN_ONLY
 
 Default: ``False``
 
-By default, when scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
+By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
 it will fail loudly if there is any ``ImportError`` exception.
 But you can choose to silence this exception and turn it into a simple
 warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
@@ -1323,6 +1447,101 @@ command.
 The project name must not conflict with the name of custom files or directories
 in the ``project`` subdirectory.
 
+.. setting:: TWISTED_REACTOR
+
+TWISTED_REACTOR
+---------------
+
+.. versionadded:: 2.0
+
+Default: ``None``
+
+Import path of a given :mod:`~twisted.internet.reactor`.
+
+Scrapy will install this reactor if no other reactor is installed yet, such as
+when the ``scrapy`` CLI program is invoked or when using the
+:class:`~scrapy.crawler.CrawlerProcess` class.
+
+If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also
+need to install the correct reactor manually. You can do that using
+:func:`~scrapy.utils.reactor.install_reactor`:
+
+.. autofunction:: scrapy.utils.reactor.install_reactor
+
+If a reactor is already installed,
+:func:`~scrapy.utils.reactor.install_reactor` has no effect.
+
+:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
+:exc:`Exception` if the installed reactor does not match the
+:setting:`TWISTED_REACTOR` setting; therfore, having top-level
+:mod:`~twisted.internet.reactor` imports in project files and imported
+third-party libraries will make Scrapy raise :exc:`Exception` when
+it checks which reactor is installed.
+
+In order to use the reactor installed by Scrapy::
+
+    import scrapy
+    from twisted.internet import reactor
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = 'quotes'
+
+        def __init__(self, *args, **kwargs):
+            self.timeout = int(kwargs.pop('timeout', '60'))
+            super(QuotesSpider, self).__init__(*args, **kwargs)
+
+        def start_requests(self):
+            reactor.callLater(self.timeout, self.stop)
+
+            urls = ['http://quotes.toscrape.com/page/1']
+            for url in urls:
+                yield scrapy.Request(url=url, callback=self.parse)
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {'text': quote.css('span.text::text').get()}
+
+        def stop(self):
+            self.crawler.engine.close_spider(self, 'timeout')
+
+
+which raises :exc:`Exception`, becomes::
+
+    import scrapy
+
+
+    class QuotesSpider(scrapy.Spider):
+        name = 'quotes'
+
+        def __init__(self, *args, **kwargs):
+            self.timeout = int(kwargs.pop('timeout', '60'))
+            super(QuotesSpider, self).__init__(*args, **kwargs)
+
+        def start_requests(self):
+            from twisted.internet import reactor
+            reactor.callLater(self.timeout, self.stop)
+
+            urls = ['http://quotes.toscrape.com/page/1']
+            for url in urls:
+                yield scrapy.Request(url=url, callback=self.parse)
+
+        def parse(self, response):
+            for quote in response.css('div.quote'):
+                yield {'text': quote.css('span.text::text').get()}
+
+        def stop(self):
+            self.crawler.engine.close_spider(self, 'timeout')
+
+
+The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
+means that Scrapy will not attempt to install any specific reactor, and the
+default reactor defined by Twisted for the current platform will be used. This
+is to maintain backward compatibility and avoid possible problems caused by
+using a non-default reactor.
+
+For additional information, see :doc:`core/howto/choosing-reactor`.
+
 
 .. setting:: URLLENGTH_LIMIT
 
@@ -1343,7 +1562,10 @@ USER_AGENT
 
 Default: ``"Scrapy/VERSION (+https://scrapy.org)"``
 
-The default User-Agent to use when crawling, unless overridden.
+The default User-Agent to use when crawling, unless overridden. This user agent is
+also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`
+if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
+there is no overridding User-Agent header specified for the request.
 
 
 Settings documented elsewhere:

docs/topics/shell.rst

@@ -31,7 +31,7 @@ for more info.
 Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
 is unavailable.
 
-Through scrapy's settings you can configure it to use any one of
+Through Scrapy's settings you can configure it to use any one of
 ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
 are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
 variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
@@ -41,7 +41,7 @@ variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
 
 .. _IPython: https://ipython.org/
 .. _IPython installation guide: https://ipython.org/install.html
-.. _bpython: https://www.bpython-interpreter.org/
+.. _bpython: https://bpython-interpreter.org/
 
 Launch the shell
 ================
@@ -142,7 +142,7 @@ Example of shell session
 ========================
 
 Here's an example of a typical shell session where we start by scraping the
-https://scrapy.org page, and then proceed to scrape the https://reddit.com
+https://scrapy.org page, and then proceed to scrape the https://old.reddit.com/
 page. Finally, we modify the (Reddit) request method to POST and re-fetch it
 getting an error. We end the session by typing Ctrl-D (in Unix systems) or
 Ctrl-Z in Windows.
@@ -177,47 +177,46 @@ all start with the ``[s]`` prefix)::
     >>>
 
 
-After that, we can start playing with the objects::
+After that, we can start playing with the objects:
 
-    >>> response.xpath('//title/text()').get()
-    'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
+>>> response.xpath('//title/text()').get()
+'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
 
-    >>> fetch("https://reddit.com")
+>>> fetch("https://old.reddit.com/")
 
-    >>> response.xpath('//title/text()').get()
-    'reddit: the front page of the internet'
+>>> response.xpath('//title/text()').get()
+'reddit: the front page of the internet'
 
-    >>> request = request.replace(method="POST")
+>>> request = request.replace(method="POST")
 
-    >>> fetch(request)
+>>> fetch(request)
 
-    >>> response.status
-    404
+>>> response.status
+404
 
-    >>> from pprint import pprint
+>>> from pprint import pprint
 
-    >>> pprint(response.headers)
-    {'Accept-Ranges': ['bytes'],
-     'Cache-Control': ['max-age=0, must-revalidate'],
-     'Content-Type': ['text/html; charset=UTF-8'],
-     'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
-     'Server': ['snooserv'],
-     'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
-                    'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
-                    'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
-                    'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
-     'Vary': ['accept-encoding'],
-     'Via': ['1.1 varnish'],
-     'X-Cache': ['MISS'],
-     'X-Cache-Hits': ['0'],
-     'X-Content-Type-Options': ['nosniff'],
-     'X-Frame-Options': ['SAMEORIGIN'],
-     'X-Moose': ['majestic'],
-     'X-Served-By': ['cache-cdg8730-CDG'],
-     'X-Timer': ['S1481214079.394283,VS0,VE159'],
-     'X-Ua-Compatible': ['IE=edge'],
-     'X-Xss-Protection': ['1; mode=block']}
-    >>>
+>>> pprint(response.headers)
+{'Accept-Ranges': ['bytes'],
+ 'Cache-Control': ['max-age=0, must-revalidate'],
+ 'Content-Type': ['text/html; charset=UTF-8'],
+ 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'],
+ 'Server': ['snooserv'],
+ 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure',
+                'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'],
+ 'Vary': ['accept-encoding'],
+ 'Via': ['1.1 varnish'],
+ 'X-Cache': ['MISS'],
+ 'X-Cache-Hits': ['0'],
+ 'X-Content-Type-Options': ['nosniff'],
+ 'X-Frame-Options': ['SAMEORIGIN'],
+ 'X-Moose': ['majestic'],
+ 'X-Served-By': ['cache-cdg8730-CDG'],
+ 'X-Timer': ['S1481214079.394283,VS0,VE159'],
+ 'X-Ua-Compatible': ['IE=edge'],
+ 'X-Xss-Protection': ['1; mode=block']}
 
 
 .. _topics-shell-inspect-response:
@@ -263,16 +262,16 @@ When you run the spider, you will get something similar to this::
     >>> response.url
     'http://example.org'
 
-Then, you can check if the extraction code is working::
+Then, you can check if the extraction code is working:
 
-    >>> response.xpath('//h1[@class="fn"]')
-    []
+>>> response.xpath('//h1[@class="fn"]')
+[]
 
 Nope, it doesn't. So you can open the response in your web browser and see if
-it's the response you were expecting::
+it's the response you were expecting:
 
-    >>> view(response)
-    True
+>>> view(response)
+True
 
 Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
 crawling::

docs/topics/signals.rst

@@ -46,14 +46,15 @@ Here is a simple example showing how you can catch signals and perform some acti
         def parse(self, response):
             pass
 
+.. _signal-deferred:
 
 Deferred signal handlers
 ========================
 
-Some signals support returning `Twisted deferreds`_ from their handlers, see
-the :ref:`topics-signals-ref` below to know which ones.
+Some signals support returning :class:`~twisted.internet.defer.Deferred`
+objects from their handlers, see the :ref:`topics-signals-ref` below to know
+which ones.
 
-.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
 
 .. _topics-signals-ref:
 
@@ -141,7 +142,7 @@ item_error
 .. signal:: item_error
 .. function:: item_error(item, response, spider, failure)
 
-    Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
+    Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
     an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
 
     This signal supports returning deferreds from their handlers.
@@ -155,8 +156,8 @@ item_error
     :param spider: the spider which raised the exception
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-    :param failure: the exception raised as a Twisted `Failure`_ object
-    :type failure: `Failure`_ object
+    :param failure: the exception raised
+    :type failure: twisted.python.failure.Failure
 
 spider_closed
 -------------
@@ -232,12 +233,12 @@ spider_error
 .. signal:: spider_error
 .. function:: spider_error(failure, response, spider)
 
-    Sent when a spider callback generates an error (ie. raises an exception).
+    Sent when a spider callback generates an error (i.e. raises an exception).
 
     This signal does not support returning deferreds from their handlers.
 
-    :param failure: the exception raised as a Twisted `Failure`_ object
-    :type failure: `Failure`_ object
+    :param failure: the exception raised
+    :type failure: twisted.python.failure.Failure
 
     :param response: the response being processed when the exception was raised
     :type response: :class:`~scrapy.http.Response` object
@@ -295,6 +296,25 @@ request_reached_downloader
     :param spider: the spider that yielded the request
     :type spider: :class:`~scrapy.spiders.Spider` object
 
+request_left_downloader
+-----------------------
+
+.. signal:: request_left_downloader
+.. function:: request_left_downloader(request, spider)
+
+    .. versionadded:: 2.0
+
+    Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of
+    failure.
+
+    This signal does not support returning deferreds from its handlers.
+
+    :param request: the request that reached the downloader
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider that yielded the request
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 response_received
 -----------------
 
@@ -333,5 +353,3 @@ response_downloaded
 
     :param spider: the spider for which the response is intended
     :type spider: :class:`~scrapy.spiders.Spider` object
-
-.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html

docs/topics/spider-middleware.rst

@@ -43,7 +43,7 @@ previous (or subsequent) middleware being applied.
 
 If you want to disable a builtin middleware (the ones defined in
 :setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
-in your project :setting:`SPIDER_MIDDLEWARES` setting and assign `None` as its
+in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
 value.  For example, if you want to disable the off-site middleware::
 
     SPIDER_MIDDLEWARES = {
@@ -54,11 +54,17 @@ value.  For example, if you want to disable the off-site middleware::
 Finally, keep in mind that some middlewares may need to be enabled through a
 particular setting. See each middleware documentation for more info.
 
+.. _custom-spider-middleware:
+
 Writing your own spider middleware
 ==================================
 
-Each middleware component is a Python class that defines one or more of the
-following methods:
+Each spider middleware is a Python class that defines one or more of the
+methods defined below.
+
+The main entry point is the ``from_crawler`` class method, which receives a
+:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler`
+object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
 .. module:: scrapy.spidermiddlewares
 
@@ -78,7 +84,8 @@ following methods:
 
         If it raises an exception, Scrapy won't bother calling any other spider
         middleware :meth:`process_spider_input` and will call the request
-        errback.  The output of the errback is chained back in the other
+        errback if there is one, otherwise it will start the :meth:`process_spider_exception`
+        chain. The output of the errback is chained back in the other
         direction for :meth:`process_spider_output` to process it, or
         :meth:`process_spider_exception` if it raised an exception.
 
@@ -112,8 +119,8 @@ following methods:
 
     .. method:: process_spider_exception(response, exception, spider)
 
-        This method is called when a spider or :meth:`process_spider_input`
-        method (from other spider middleware) raises an exception.
+        This method is called when a spider or :meth:`process_spider_output`
+        method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
         iterable of :class:`~scrapy.http.Request`, dict or
@@ -125,7 +132,8 @@ following methods:
         exception reaches the engine (where it's logged and discarded).
 
         If it returns an iterable the :meth:`process_spider_output` pipeline
-        kicks in, and no other :meth:`process_spider_exception` will be called.
+        kicks in, starting from the next spider middleware, and no other
+        :meth:`process_spider_exception` will be called.
 
         :param response: the response being processed when the exception was
           raised
@@ -200,7 +208,7 @@ DepthMiddleware
 .. class:: DepthMiddleware
 
    DepthMiddleware is used for tracking the depth of each Request inside the
-   site being scraped. It works by setting `request.meta['depth'] = 0` whenever
+   site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
    there is no value previously set (usually just the first Request) and
    incrementing it by 1 otherwise.
 

docs/topics/spiders.rst

@@ -72,8 +72,6 @@ scrapy.Spider
        spider that crawls ``mywebsite.com`` would often be called
        ``mywebsite``.
 
-       .. note:: In Python 2 this must be ASCII only.
-
    .. attribute:: allowed_domains
 
        An optional list of strings containing domains that this spider is
@@ -129,7 +127,7 @@ scrapy.Spider
 
        You probably won't need to override this directly because the default
        implementation acts as a proxy to the :meth:`__init__` method, calling
-       it with the given arguments `args` and named arguments `kwargs`.
+       it with the given arguments ``args`` and named arguments ``kwargs``.
 
        Nonetheless, this method sets the :attr:`crawler` and :attr:`settings`
        attributes in the new instance so they can be accessed later inside the
@@ -190,7 +188,7 @@ scrapy.Spider
    .. method:: log(message, [level, component])
 
        Wrapper that sends a log message through the Spider's :attr:`logger`,
-       kept for backwards compatibility. For more information see
+       kept for backward compatibility. For more information see
        :ref:`topics-logging-from-spiders`.
 
    .. method:: closed(reason)
@@ -298,13 +296,13 @@ The above example can also be written as follows::
 
 Keep in mind that spider arguments are only strings.
 The spider will not do any parsing on its own.
-If you were to set the `start_urls` attribute from the command line,
+If you were to set the ``start_urls`` attribute from the command line,
 you would have to parse it on your own into a list
 using something like
-`ast.literal_eval <https://docs.python.org/library/ast.html#ast.literal_eval>`_
-or `json.loads <https://docs.python.org/library/json.html#json.loads>`_
+`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
+or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
 and then set it as an attribute.
-Otherwise, you would cause iteration over a `start_urls` string
+Otherwise, you would cause iteration over a ``start_urls`` string
 (a very common python pitfall)
 resulting in each character being seen as a separate url.
 
@@ -374,16 +372,23 @@ CrawlSpider
 Crawling rules
 ~~~~~~~~~~~~~~
 
-.. class:: Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None)
+.. autoclass:: Rule
 
    ``link_extractor`` is a :ref:`Link Extractor <topics-link-extractors>` object which
-   defines how links will be extracted from each crawled page.
+   defines how links will be extracted from each crawled page. Each produced link will
+   be used to generate a :class:`~scrapy.http.Request` object, which will contain the
+   link's text in its ``meta`` dictionary (under the ``link_text`` key).
+   If omitted, a default link extractor created with no arguments will be used,
+   resulting in all links being extracted.
 
    ``callback`` is a callable or a string (in which case a method from the spider
    object with that name will be used) to be called for each link extracted with
-   the specified link_extractor. This callback receives a response as its first
-   argument and must return a list containing :class:`~scrapy.item.Item` and/or
-   :class:`~scrapy.http.Request` objects (or any subclass of them).
+   the specified link extractor. This callback receives a :class:`~scrapy.http.Response`
+   as its first argument and must return either a single instance or an iterable of
+   :class:`~scrapy.item.Item`, ``dict`` and/or :class:`~scrapy.http.Request` objects
+   (or any subclass of them). As mentioned above, the received :class:`~scrapy.http.Response`
+   object will contain the text of the link that produced the :class:`~scrapy.http.Request`
+   in its ``meta`` dictionary (under the ``link_text`` key)
 
    .. warning:: When writing crawl spider rules, avoid using ``parse`` as
        callback, since the :class:`CrawlSpider` uses the ``parse`` method
@@ -402,10 +407,21 @@ Crawling rules
    of links extracted from each response using the specified ``link_extractor``.
    This is mainly used for filtering purposes.
 
-   ``process_request`` is a callable, or a string (in which case a method from
-   the spider object with that name will be used) which will be called with
-   every request extracted by this rule, and must return a request or None (to
-   filter out the request).
+   ``process_request`` is a callable (or a string, in which case a method from
+   the spider object with that name will be used) which will be called for every
+   :class:`~scrapy.http.Request` extracted by this rule. This callable should
+   take said request as first argument and the :class:`~scrapy.http.Response`
+   from which the request originated as second argument. It must return a
+   ``Request`` object or ``None`` (to filter out the request).
+
+   ``errback`` is a callable or a string (in which case a method from the spider
+   object with that name will be used) to be called if any exception is
+   raised while processing a request generated by the rule.
+   It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
+   instance as first parameter.
+
+   .. versionadded:: 2.0
+      The *errback* parameter.
 
 CrawlSpider example
 ~~~~~~~~~~~~~~~~~~~
@@ -436,6 +452,7 @@ Let's now take a look at an example CrawlSpider with rules::
             item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
             item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
             item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
+            item['link_text'] = response.meta['link_text']
             return item
 
 
@@ -655,7 +672,7 @@ SitemapSpider
 
     .. attribute:: sitemap_follow
 
-        A list of regexes of sitemap that should be followed. This is is only
+        A list of regexes of sitemap that should be followed. This is only
         for sites that use `Sitemap index files`_ that point to other sitemap
         files.
 
@@ -680,6 +697,50 @@ SitemapSpider
 
         Default is ``sitemap_alternate_links`` disabled.
 
+    .. method:: sitemap_filter(entries)
+
+        This is a filter function that could be overridden to select sitemap entries
+        based on their attributes.
+
+        For example::
+
+            <url>
+                <loc>http://example.com/</loc>
+                <lastmod>2005-01-01</lastmod>
+            </url>
+
+        We can define a ``sitemap_filter`` function to filter ``entries`` by date::
+
+            from datetime import datetime
+            from scrapy.spiders import SitemapSpider
+
+            class FilteredSitemapSpider(SitemapSpider):
+                name = 'filtered_sitemap_spider'
+                allowed_domains = ['example.com']
+                sitemap_urls = ['http://example.com/sitemap.xml']
+
+                def sitemap_filter(self, entries):
+                    for entry in entries:
+                        date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
+                        if date_time.year >= 2005:
+                            yield entry
+
+        This would retrieve only ``entries`` modified on 2005 and the following
+        years.
+
+        Entries are dict objects extracted from the sitemap document.
+        Usually, the key is the tag name and the value is the text inside it.
+
+        It's important to notice that:
+
+        - as the loc attribute is required, entries without this tag are discarded
+        - alternate links are stored in a list with the key ``alternate``
+          (see ``sitemap_alternate_links``)
+        - namespaces are removed, so lxml tags named as ``{namespace}tagname`` become only ``tagname``
+
+        If you omit this method, all entries found in sitemaps will be
+        processed, observing other attributes and their settings.
+
 
 SitemapSpider examples
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -753,6 +814,6 @@ Combine SitemapSpider with other sources of urls::
 
 .. _Sitemaps: https://www.sitemaps.org/index.html
 .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
-.. _robots.txt: http://www.robotstxt.org/
+.. _robots.txt: https://www.robotstxt.org/
 .. _TLD: https://en.wikipedia.org/wiki/Top-level_domain
 .. _Scrapyd documentation: https://scrapyd.readthedocs.io/en/latest/

docs/topics/stats.rst

@@ -32,7 +32,7 @@ Common Stats Collector uses
 Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats`
 attribute. Here is an example of an extension that access stats::
 
-    class ExtensionThatAccessStats(object):
+    class ExtensionThatAccessStats:
 
         def __init__(self, stats):
             self.stats = stats
@@ -57,15 +57,15 @@ Set stat value only if lower than previous::
 
     stats.min_value('min_free_memory_percent', value)
 
-Get stat value::
+Get stat value:
 
-    >>> stats.get_value('custom_count')
-    1
+>>> stats.get_value('custom_count')
+1
 
-Get all stats::
+Get all stats:
 
-    >>> stats.get_stats()
-    {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
+>>> stats.get_stats()
+{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
 
 Available Stats Collectors
 ==========================
@@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select
 which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
 default Stats Collector used is the :class:`MemoryStatsCollector`. 
 
-.. module:: scrapy.statscollectors
-   :synopsis: Stats Collectors
+.. currentmodule:: scrapy.statscollectors
 
 MemoryStatsCollector
 --------------------

docs/topics/telnetconsole.rst

@@ -1,12 +1,11 @@
+.. currentmodule:: scrapy.extensions.telnet
+
 .. _topics-telnetconsole:
 
 ==============
 Telnet Console
 ==============
 
-.. module:: scrapy.extensions.telnet
-   :synopsis: The Telnet Console
-
 Scrapy comes with a built-in telnet console for inspecting and controlling a
 Scrapy running process. The telnet console is just a regular python shell
 running inside the Scrapy process, so you can do literally anything from it.
@@ -16,6 +15,17 @@ The telnet console is a :ref:`built-in Scrapy extension
 disable it if you want. For more information about the extension itself see
 :ref:`topics-extensions-ref-telnetconsole`.
 
+.. warning::
+    It is not secure to use telnet console via public networks, as telnet
+    doesn't provide any transport-layer security. Having username/password
+    authentication doesn't change that.
+
+    Intended usage is connecting to a running Scrapy spider locally
+    (spider process and telnet client are on the same machine)
+    or over a secure connection (VPN, SSH tunnel).
+    Please avoid using telnet console over insecure connections,
+    or disable it completely using :setting:`TELNETCONSOLE_ENABLED` option.
+
 .. highlight:: none
 
 How to access the telnet console
@@ -26,8 +36,26 @@ The telnet console listens in the TCP port defined in the
 the console you need to type::
 
     telnet localhost 6023
+    Trying localhost...
+    Connected to localhost.
+    Escape character is '^]'.
+    Username:
+    Password:  
     >>>
-    
+
+By default Username is ``scrapy`` and Password is autogenerated. The 
+autogenerated Password can be seen on Scrapy logs like the example below::
+
+    2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
+
+Default Username and Password can be overridden by the settings
+:setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
+
+.. warning::
+    Username and password provide only a limited protection, as telnet
+    is not using secure transport - by default traffic is not encrypted
+    even if username and password are set.
+
 You need the telnet program which comes installed by default in Windows, and
 most Linux distros.
 
@@ -160,3 +188,24 @@ Default: ``'127.0.0.1'``
 
 The interface the telnet console should listen on
 
+
+.. setting:: TELNETCONSOLE_USERNAME
+
+TELNETCONSOLE_USERNAME
+----------------------
+
+Default: ``'scrapy'``
+
+The username used for the telnet console
+
+
+.. setting:: TELNETCONSOLE_PASSWORD
+
+TELNETCONSOLE_PASSWORD
+----------------------
+
+Default: ``None``
+
+The password used for the telnet console, default behaviour is to have it
+autogenerated
+

docs/topics/ubuntu.rst

@@ -1,41 +0,0 @@
-:orphan: Ubuntu packages are obsolete
-
-.. _topics-ubuntu:
-
-===============
-Ubuntu packages
-===============
-
-.. versionadded:: 0.10
-
-`Scrapinghub`_ publishes apt-gettable packages which are generally fresher than
-those in Ubuntu, and more stable too since they're continuously built from
-`GitHub repo`_ (master & stable branches) and so they contain the latest bug
-fixes.
-
-.. caution:: These packages are currently not updated and may not work on
-   Ubuntu 16.04 and above, see :issue:`2076` and :issue:`2137`.
-
-To use the packages:
-
-1. Import the GPG key used to sign Scrapy packages into APT keyring::
-
-    sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
-
-2. Create `/etc/apt/sources.list.d/scrapy.list` file using the following command::
-
-    echo 'deb http://archive.scrapy.org/ubuntu scrapy main' | sudo tee /etc/apt/sources.list.d/scrapy.list
-
-3. Update package lists and install the scrapy package:
-
-   .. parsed-literal::
-
-      sudo apt-get update && sudo apt-get install scrapy
-
-.. note:: Repeat step 3 if you are trying to upgrade Scrapy.
-
-.. warning:: `python-scrapy` is a different package provided by official debian
-   repositories, it's very outdated and it isn't supported by Scrapy team.
-
-.. _Scrapinghub: https://scrapinghub.com/
-.. _GitHub repo: https://github.com/scrapy/scrapy

docs/versioning.rst

@@ -12,7 +12,7 @@ There are 3 numbers in a Scrapy version: *A.B.C*
 * *A* is the major version. This will rarely change and will signify very
   large changes.
 * *B* is the release number. This will include many changes including features
-  and things that possibly break backwards compatibility, although we strive to
+  and things that possibly break backward compatibility, although we strive to
   keep theses cases at a minimum.
 * *C* is the bugfix release number.
 

extras/qps-bench-server.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 from time import time
 from collections import deque
 from twisted.web.server import Site, NOT_DONE_YET

extras/scrapy_zsh_completion

@@ -1,25 +1,213 @@
 #compdef scrapy
-
-# zsh completion for the Scrapy command-line tool
-
 _scrapy() {
-    local curcontext="$curcontext" cmd spiders
+    local context state state_descr line
+    local ret=1
     typeset -A opt_args
-    cmd=$words[2]
-    
-    case "$cmd" in
-        crawl|edit|check)
-            spiders=$(scrapy list 2>/dev/null) || spiders=""
-            if [[ -n "$spiders" ]]; then
-                compadd `echo $spiders`
-            fi
-        ;;
-        *)
-            if [[ CURRENT -eq 2 ]]; then
-                _arguments '*: :(check crawl edit fetch genspider list parse runspider settings shell startproject version view)'
-            fi
-        ;;
+    _arguments \
+	"(- 1 *)"{-h,--help}"[Help]" \
+	"1: :->command" \
+	"*:: :->args" && ret=0
+
+    case $state in
+	command)
+	    _scrapy_cmds
+	    ;;
+	args)
+	    case $words[1] in
+		(bench)
+		    _scrapy_glb_opts
+		    ;;
+		(fetch)
+		    local options=(
+			'--headers[print response HTTP headers instead of body]'
+			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+			'--spider=[use this spider]:spider:_scrapy_spiders'
+			'1::URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(genspider)
+		    local options=(
+			{'(--list)-l','(-l)--list'}'[List available templates]'
+			{'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]'
+			'--force[If the spider already exists, overwrite it with the template]'
+			{'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)'
+			{'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)'
+			'1:name:(NAME)'
+			'2:domain:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(runspider)
+		    local options=(
+			{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+			{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
+			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+			'1:spider file:_files -g \*.py'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(settings)
+		    local options=(
+			'--get=[print raw setting value]:option:(SETTING)'
+			'--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)'
+			'--getint=[print setting value, interpreted as an integer]:option:(SETTING)'
+			'--getfloat=[print setting value, interpreted as a float]:option:(SETTING)'
+			'--getlist=[print setting value, interpreted as a list]:option:(SETTING)'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(shell)
+		    local options=(
+			'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
+			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+			'--spider=[use this spider]:spider:_scrapy_spiders'
+			'::file:_files -g \*.html'
+			'::URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(startproject)
+		    local options=(
+			'1:name:(NAME)'
+			'2:dir:_dir_list'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(version)
+		    local options=(
+			{'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(view)
+		    local options=(
+			'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
+			'--spider=[use this spider]:spider:_scrapy_spiders'
+			'1:URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(check)
+		    local options=(
+			{'(--list)-l','(-l)--list'}'[only list contracts, without checking them]'
+			{'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]'
+			'1:spider:_scrapy_spiders'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(crawl)
+		    local options=(
+			{'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files'
+			{'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)'
+			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+			'1:spider:_scrapy_spiders'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(edit)
+		    local options=(
+		      '1:spider:_scrapy_spiders'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+		(list)
+		    _scrapy_glb_opts
+		    ;;
+		(parse)
+		    local options=(
+			'*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)'
+			'--spider=[use this spider without looking for one]:spider:_scrapy_spiders'
+			'--pipelines[process items through pipelines]'
+			"--nolinks[don't show links to follow (extracted requests)]"
+			"--noitems[don't show scraped items]"
+			'--nocolour[avoid using pygments to colorize the output]'
+			{'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]'
+			{'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)'
+			{'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)'
+			'--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)'
+			{'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)'
+			{'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]'
+			'1:URL:_httpie_urls'
+		    )
+		    _scrapy_glb_opts $options
+		    ;;
+	    esac
+	    ;;
     esac
+
+    return ret
 }
 
-_scrapy
+_scrapy_cmds() {
+    local -a commands project_commands
+    commands=(
+        'bench:Run quick benchmark test'
+        'fetch:Fetch a URL using the Scrapy downloader'
+        'genspider:Generate new spider using pre-defined templates'
+        'runspider:Run a self-contained spider (without creating a project)'
+        'settings:Get settings values'
+        'shell:Interactive scraping console'
+        'startproject:Create new project'
+        'version:Print Scrapy version'
+        'view:Open URL in browser, as seen by Scrapy'
+    )
+    project_commands=(
+	'check:Check spider contracts'
+	'crawl:Run a spider'
+        'edit:Edit spider'
+        'list:List available spiders'
+        'parse:Parse URL (using its spider) and print the results'
+    )
+    if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+	commands=(${commands[@]} ${project_commands[@]})
+    fi
+    _describe -t common-commands 'common commands' commands && ret=0
+}
+
+_scrapy_glb_opts() {
+    local -a options
+    options=(
+	'(- *)'{-h,--help}'[show this help message and exit]'
+	'(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files'
+	'--pidfile=[write process ID to FILE]:file:_files'
+	'--profile=[write python cProfile stats to FILE]:file:_files'
+	{'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)'
+	'(-L --loglevel --logfile)--nolog[disable logging completely]'
+	'--pdb[enable pdb on failure]'
+	'*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)'
+    )
+    options=(${options[@]} "$@")
+    _arguments -A "-*" $options && ret=0
+}
+
+_httpie_urls() {
+
+  local ret=1
+
+  if ! [[ -prefix [-+.a-z0-9]#:// ]]; then
+    local expl
+    compset -S '[^:/]*' && compstate[to_end]=''
+    _wanted url-schemas expl 'URL schema' compadd -S '' http:// https:// && ret=0
+  else
+    _urls && ret=0
+  fi
+
+  return $ret
+
+}
+
+_scrapy_spiders() {
+
+  local ret=1
+
+  if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then
+      compadd -S '' $(scrapy list) && ret=0
+  else
+      compadd -S '' SPIDER && ret=0
+  fi
+
+  return $ret
+}
+
+_scrapy $@

pytest.ini

@@ -2,5 +2,250 @@
 usefixtures = chdir
 python_files=test_*.py __init__.py
 python_classes=
-addopts = --doctest-modules --assert=plain
+addopts =
+    --assert=plain
+    --doctest-modules
+    --ignore=docs/_ext
+    --ignore=docs/conf.py
+    --ignore=docs/news.rst
+    --ignore=docs/topics/dynamic-content.rst
+    --ignore=docs/topics/items.rst
+    --ignore=docs/topics/leaks.rst
+    --ignore=docs/topics/loaders.rst
+    --ignore=docs/topics/selectors.rst
+    --ignore=docs/topics/shell.rst
+    --ignore=docs/topics/stats.rst
+    --ignore=docs/topics/telnetconsole.rst
+    --ignore=docs/utils
 twisted = 1
+markers =
+    only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+flake8-ignore =
+    W503
+    # Files that are only meant to provide top-level imports are expected not
+    # to use any of their imports:
+    scrapy/core/downloader/handlers/http.py F401
+    scrapy/http/__init__.py F401
+    # Issues pending a review:
+    # extras
+    extras/qps-bench-server.py E501
+    extras/qpsclient.py E501 E501
+    # scrapy/commands
+    scrapy/commands/__init__.py E128 E501
+    scrapy/commands/check.py E501
+    scrapy/commands/crawl.py E501
+    scrapy/commands/edit.py E501
+    scrapy/commands/fetch.py E401 E501 E128 E731
+    scrapy/commands/genspider.py E128 E501
+    scrapy/commands/parse.py E128 E501 E731
+    scrapy/commands/runspider.py E501
+    scrapy/commands/settings.py E128
+    scrapy/commands/shell.py E128 E501
+    scrapy/commands/startproject.py E127 E501 E128
+    scrapy/commands/version.py E501 E128
+    # scrapy/contracts
+    scrapy/contracts/__init__.py E501 W504
+    scrapy/contracts/default.py E128
+    # scrapy/core
+    scrapy/core/engine.py E501 E128 E127
+    scrapy/core/scheduler.py E501
+    scrapy/core/scraper.py E501 E128 W504
+    scrapy/core/spidermw.py E501 E731 E126
+    scrapy/core/downloader/__init__.py E501
+    scrapy/core/downloader/contextfactory.py E501 E128 E126
+    scrapy/core/downloader/middleware.py E501
+    scrapy/core/downloader/tls.py E501
+    scrapy/core/downloader/webclient.py E731 E501 E128 E126
+    scrapy/core/downloader/handlers/__init__.py E501
+    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
+    scrapy/core/downloader/handlers/http10.py E501
+    scrapy/core/downloader/handlers/http11.py E501
+    scrapy/core/downloader/handlers/s3.py E501 E128 E126
+    # scrapy/downloadermiddlewares
+    scrapy/downloadermiddlewares/ajaxcrawl.py E501
+    scrapy/downloadermiddlewares/decompression.py E501
+    scrapy/downloadermiddlewares/defaultheaders.py E501
+    scrapy/downloadermiddlewares/httpcache.py E501 E126
+    scrapy/downloadermiddlewares/httpcompression.py E501 E128
+    scrapy/downloadermiddlewares/httpproxy.py E501
+    scrapy/downloadermiddlewares/redirect.py E501 W504
+    scrapy/downloadermiddlewares/retry.py E501 E126
+    scrapy/downloadermiddlewares/robotstxt.py E501
+    scrapy/downloadermiddlewares/stats.py E501
+    # scrapy/extensions
+    scrapy/extensions/closespider.py E501 E128 E123
+    scrapy/extensions/corestats.py E501
+    scrapy/extensions/feedexport.py E128 E501
+    scrapy/extensions/httpcache.py E128 E501
+    scrapy/extensions/memdebug.py E501
+    scrapy/extensions/spiderstate.py E501
+    scrapy/extensions/telnet.py E501 W504
+    scrapy/extensions/throttle.py E501
+    # scrapy/http
+    scrapy/http/common.py E501
+    scrapy/http/cookies.py E501
+    scrapy/http/request/__init__.py E501
+    scrapy/http/request/form.py E501 E123
+    scrapy/http/request/json_request.py E501
+    scrapy/http/response/__init__.py E501 E128
+    scrapy/http/response/text.py E501 E128 E124
+    # scrapy/linkextractors
+    scrapy/linkextractors/__init__.py E731 E501 E402 W504
+    scrapy/linkextractors/lxmlhtml.py E501 E731
+    # scrapy/loader
+    scrapy/loader/__init__.py E501 E128
+    scrapy/loader/processors.py E501
+    # scrapy/pipelines
+    scrapy/pipelines/__init__.py E501
+    scrapy/pipelines/files.py E116 E501
+    scrapy/pipelines/images.py E501
+    scrapy/pipelines/media.py E125 E501
+    # scrapy/selector
+    scrapy/selector/__init__.py F403
+    scrapy/selector/unified.py E501 E111
+    # scrapy/settings
+    scrapy/settings/__init__.py E501
+    scrapy/settings/default_settings.py E501 E114 E116
+    scrapy/settings/deprecated.py E501
+    # scrapy/spidermiddlewares
+    scrapy/spidermiddlewares/httperror.py E501
+    scrapy/spidermiddlewares/offsite.py E501
+    scrapy/spidermiddlewares/referer.py E501 E129 W504
+    scrapy/spidermiddlewares/urllength.py E501
+    # scrapy/spiders
+    scrapy/spiders/__init__.py E501 E402
+    scrapy/spiders/crawl.py E501
+    scrapy/spiders/feed.py E501
+    scrapy/spiders/sitemap.py E501
+    # scrapy/utils
+    scrapy/utils/asyncio.py E501
+    scrapy/utils/benchserver.py E501
+    scrapy/utils/conf.py E402 E501
+    scrapy/utils/datatypes.py E501
+    scrapy/utils/decorators.py E501
+    scrapy/utils/defer.py E501 E128
+    scrapy/utils/deprecate.py E128 E501 E127
+    scrapy/utils/gz.py E501 W504
+    scrapy/utils/http.py F403
+    scrapy/utils/httpobj.py E501
+    scrapy/utils/iterators.py E501
+    scrapy/utils/log.py E128 E501
+    scrapy/utils/markup.py F403
+    scrapy/utils/misc.py E501
+    scrapy/utils/multipart.py F403
+    scrapy/utils/project.py E501
+    scrapy/utils/python.py E501
+    scrapy/utils/reactor.py E501
+    scrapy/utils/reqser.py E501
+    scrapy/utils/request.py E127 E501
+    scrapy/utils/response.py E501 E128
+    scrapy/utils/signal.py E501 E128
+    scrapy/utils/sitemap.py E501
+    scrapy/utils/spider.py E501
+    scrapy/utils/ssl.py E501
+    scrapy/utils/test.py E501
+    scrapy/utils/url.py E501 F403 E128 F405
+    # scrapy
+    scrapy/__init__.py E402 E501
+    scrapy/cmdline.py E501
+    scrapy/crawler.py E501
+    scrapy/dupefilters.py E501
+    scrapy/exceptions.py E501
+    scrapy/exporters.py E501
+    scrapy/interfaces.py E501
+    scrapy/item.py E501 E128
+    scrapy/link.py E501
+    scrapy/logformatter.py E501
+    scrapy/mail.py E402 E128 E501
+    scrapy/middleware.py E128 E501
+    scrapy/pqueues.py E501
+    scrapy/resolver.py E501
+    scrapy/responsetypes.py E128 E501
+    scrapy/robotstxt.py E501
+    scrapy/shell.py E501
+    scrapy/signalmanager.py E501
+    scrapy/spiderloader.py F841 E501 E126
+    scrapy/squeues.py E128
+    scrapy/statscollectors.py E501
+    # tests
+    tests/__init__.py E402 E501
+    tests/mockserver.py E401 E501 E126 E123
+    tests/pipelines.py F841
+    tests/spiders.py E501 E127
+    tests/test_closespider.py E501 E127
+    tests/test_command_fetch.py E501
+    tests/test_command_parse.py E501 E128
+    tests/test_command_shell.py E501 E128
+    tests/test_commands.py E128 E501
+    tests/test_contracts.py E501 E128
+    tests/test_crawl.py E501 E741
+    tests/test_crawler.py F841 E501
+    tests/test_dependencies.py F841 E501
+    tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
+    tests/test_downloadermiddleware.py E501
+    tests/test_downloadermiddleware_ajaxcrawlable.py E501
+    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E126
+    tests/test_downloadermiddleware_decompression.py E127
+    tests/test_downloadermiddleware_defaultheaders.py E501
+    tests/test_downloadermiddleware_downloadtimeout.py E501
+    tests/test_downloadermiddleware_httpcache.py E501
+    tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
+    tests/test_downloadermiddleware_httpproxy.py E501 E128
+    tests/test_downloadermiddleware_redirect.py E501 E128 E127
+    tests/test_downloadermiddleware_retry.py E501 E128 E126
+    tests/test_downloadermiddleware_robotstxt.py E501
+    tests/test_downloadermiddleware_stats.py E501
+    tests/test_dupefilters.py E501 E741 E128 E124
+    tests/test_engine.py E401 E501 E128
+    tests/test_exporters.py E501 E731 E128 E124
+    tests/test_extension_telnet.py F841
+    tests/test_feedexport.py E501 F841
+    tests/test_http_cookies.py E501
+    tests/test_http_headers.py E501
+    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
+    tests/test_http_response.py E501 E128
+    tests/test_item.py E128 F841
+    tests/test_link.py E501
+    tests/test_linkextractors.py E501 E128 E124
+    tests/test_loader.py E501 E731 E741 E128 E117
+    tests/test_logformatter.py E128 E501 E122
+    tests/test_mail.py E128 E501
+    tests/test_middleware.py E501 E128
+    tests/test_pipeline_crawl.py E501 E128 E126
+    tests/test_pipeline_files.py E501
+    tests/test_pipeline_images.py F841 E501
+    tests/test_pipeline_media.py E501 E741 E731 E128
+    tests/test_proxy_connect.py E501 E741
+    tests/test_request_cb_kwargs.py E501
+    tests/test_responsetypes.py E501
+    tests/test_robotstxt_interface.py E501 E501
+    tests/test_scheduler.py E501 E126 E123
+    tests/test_selector.py E501 E127
+    tests/test_spider.py E501
+    tests/test_spidermiddleware.py E501
+    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
+    tests/test_spidermiddleware_offsite.py E501 E128 E111
+    tests/test_spidermiddleware_output_chain.py E501
+    tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
+    tests/test_squeues.py E501 E741
+    tests/test_utils_asyncio.py E501
+    tests/test_utils_conf.py E501 E128
+    tests/test_utils_curl.py E501
+    tests/test_utils_datatypes.py E402 E501
+    tests/test_utils_defer.py E501 F841
+    tests/test_utils_deprecate.py F841 E501
+    tests/test_utils_http.py E501 E128 W504
+    tests/test_utils_iterators.py E501 E128 E129
+    tests/test_utils_log.py E741
+    tests/test_utils_python.py E501 E731
+    tests/test_utils_reqser.py E501 E128
+    tests/test_utils_request.py E501 E128
+    tests/test_utils_response.py E501
+    tests/test_utils_signal.py E741 F841 E731
+    tests/test_utils_sitemap.py E128 E501 E124
+    tests/test_utils_url.py E501 E127 E125 E501 E126 E123
+    tests/test_webclient.py E501 E128 E122 E402 E123 E126
+    tests/test_cmdline/__init__.py E501
+    tests/test_settings/__init__.py E501 E128
+    tests/test_spiderloader/__init__.py E128 E501
+    tests/test_utils_misc/__init__.py E501

requirements-py2.txt

@@ -1,10 +0,0 @@
-Twisted>=13.1.0
-lxml
-pyOpenSSL
-cssselect>=0.9
-queuelib
-w3lib>=1.17.0
-six>=1.5.2
-PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity

requirements-py3.txt

@@ -1,10 +0,0 @@
-Twisted>=17.9.0
-lxml>=3.2.4
-pyOpenSSL>=0.13.1
-cssselect>=0.9
-queuelib>=1.1.1
-w3lib>=1.17.0
-six>=1.5.2
-PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity

scrapy/VERSION

@@ -1 +1 @@
-1.5.0
+2.0.0

scrapy/__init__.py

@@ -14,8 +14,8 @@ del pkgutil
 
 # Check minimum required Python version
 import sys
-if sys.version_info < (2, 7):
-    print("Scrapy %s requires Python 2.7" % __version__)
+if sys.version_info < (3, 5):
+    print("Scrapy %s requires Python 3.5" % __version__)
     sys.exit(1)
 
 # Ignore noisy twisted deprecation warnings
@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
 # Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
 del _monkeypatches
 
 from twisted import version as _txv

scrapy/_monkeypatches.py

@@ -1,18 +1,4 @@
-import sys
-from six.moves import copyreg
-
-if sys.version_info[0] == 2:
-    from urlparse import urlparse
-
-    # workaround for https://bugs.python.org/issue7904 - Python < 2.7
-    if urlparse('s3://bucket/key').netloc != 'bucket':
-        from urlparse import uses_netloc
-        uses_netloc.append('s3')
-
-    # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
-    if urlparse('s3://bucket/key?key=value').query != 'key=value':
-        from urlparse import uses_query
-        uses_query.append('s3')
+import copyreg
 
 
 # Undo what Twisted's perspective broker adds to pickle register

scrapy/cmdline.py

@@ -1,5 +1,5 @@
-from __future__ import print_function
-import sys, os
+import sys
+import os
 import optparse
 import cProfile
 import inspect
@@ -12,7 +12,7 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
-from scrapy.settings.deprecated import check_deprecated_settings
+
 
 def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
@@ -25,6 +25,7 @@ def _iter_command_classes(module_name):
                     not obj == ScrapyCommand:
                 yield obj
 
+
 def _get_commands_from_module(module, inproject):
     d = {}
     for cmd in _iter_command_classes(module):
@@ -33,6 +34,7 @@ def _get_commands_from_module(module, inproject):
             d[cmdname] = cmd()
     return d
 
+
 def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
     cmds = {}
     for entry_point in pkg_resources.iter_entry_points(group):
@@ -43,6 +45,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
             raise Exception("Invalid entry point %s" % entry_point.name)
     return cmds
 
+
 def _get_commands_dict(settings, inproject):
     cmds = _get_commands_from_module('scrapy.commands', inproject)
     cmds.update(_get_commands_from_entry_points(inproject))
@@ -51,6 +54,7 @@ def _get_commands_dict(settings, inproject):
         cmds.update(_get_commands_from_module(cmds_module, inproject))
     return cmds
 
+
 def _pop_command_name(argv):
     i = 0
     for arg in argv[1:]:
@@ -59,13 +63,15 @@ def _pop_command_name(argv):
             return arg
         i += 1
 
+
 def _print_header(settings, inproject):
     if inproject:
-        print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
-            settings['BOT_NAME']))
+        print("Scrapy %s - project: %s\n" % (scrapy.__version__,
+                                             settings['BOT_NAME']))
     else:
         print("Scrapy %s - no active project\n" % scrapy.__version__)
 
+
 def _print_commands(settings, inproject):
     _print_header(settings, inproject)
     print("Usage:")
@@ -80,11 +86,13 @@ def _print_commands(settings, inproject):
     print()
     print('Use "scrapy <command> -h" to see more info about a command')
 
+
 def _print_unknown_command(settings, cmdname, inproject):
     _print_header(settings, inproject)
     print("Unknown command: %s\n" % cmdname)
     print('Use "scrapy" to see available commands')
 
+
 def _run_print_help(parser, func, *a, **kw):
     try:
         func(*a, **kw)
@@ -95,41 +103,26 @@ def _run_print_help(parser, func, *a, **kw):
             parser.print_help()
         sys.exit(2)
 
+
 def execute(argv=None, settings=None):
     if argv is None:
         argv = sys.argv
 
-    # --- backwards compatibility for scrapy.conf.settings singleton ---
-    if settings is None and 'scrapy.conf' in sys.modules:
-        from scrapy import conf
-        if hasattr(conf, 'settings'):
-            settings = conf.settings
-    # ------------------------------------------------------------------
-
     if settings is None:
         settings = get_project_settings()
         # set EDITOR from environment if available
         try:
             editor = os.environ['EDITOR']
-        except KeyError: pass
+        except KeyError:
+            pass
         else:
             settings['EDITOR'] = editor
-    check_deprecated_settings(settings)
-
-    # --- backwards compatibility for scrapy.conf.settings singleton ---
-    import warnings
-    from scrapy.exceptions import ScrapyDeprecationWarning
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-        from scrapy import conf
-        conf.settings = settings
-    # ------------------------------------------------------------------
 
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
-    parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
-        conflict_handler='resolve')
+    parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
+                                   conflict_handler='resolve')
     if not cmdname:
         _print_commands(settings, inproject)
         sys.exit(0)
@@ -150,12 +143,14 @@ def execute(argv=None, settings=None):
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
 
+
 def _run_command(cmd, args, opts):
     if opts.profile:
         _run_command_profiled(cmd, args, opts)
     else:
         cmd.run(args, opts)
 
+
 def _run_command_profiled(cmd, args, opts):
     if opts.profile:
         sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
@@ -165,6 +160,7 @@ def _run_command_profiled(cmd, args, opts):
     if opts.profile:
         p.dump_stats(opts.profile)
 
+
 if __name__ == '__main__':
     try:
         execute()

scrapy/commands/__init__.py

@@ -9,7 +9,7 @@ from scrapy.utils.conf import arglist_to_dict
 from scrapy.exceptions import UsageError
 
 
-class ScrapyCommand(object):
+class ScrapyCommand:
 
     requires_project = False
     crawler_process = None
@@ -47,7 +47,7 @@ class ScrapyCommand(object):
 
     def help(self):
         """An extensive help for the command. It will be shown when using the
-        "help" command. It can contain newlines, since not post-formatting will
+        "help" command. It can contain newlines, since no post-formatting will
         be applied to its contents.
         """
         return self.long_desc()

scrapy/commands/bench.py

@@ -1,8 +1,7 @@
 import sys
 import time
 import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -26,7 +25,7 @@ class Command(ScrapyCommand):
             self.crawler_process.start()
 
 
-class _BenchServer(object):
+class _BenchServer:
 
     def __enter__(self):
         from scrapy.utils.test import get_testenv

scrapy/commands/check.py

@@ -1,12 +1,10 @@
-from __future__ import print_function
 import time
-import sys
 from collections import defaultdict
 from unittest import TextTestRunner, TextTestResult as _TextTestResult
 
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, set_environ
 from scrapy.utils.conf import build_component_list
 
 
@@ -68,16 +66,17 @@ class Command(ScrapyCommand):
 
         spider_loader = self.crawler_process.spider_loader
 
-        for spidername in args or spider_loader.list():
-            spidercls = spider_loader.load(spidername)
-            spidercls.start_requests = lambda s: conman.from_spider(s, result)
+        with set_environ(SCRAPY_CHECK='true'):
+            for spidername in args or spider_loader.list():
+                spidercls = spider_loader.load(spidername)
+                spidercls.start_requests = lambda s: conman.from_spider(s, result)
 
-            tested_methods = conman.tested_methods_from_spidercls(spidercls)
-            if opts.list:
-                for method in tested_methods:
-                    contract_reqs[spidercls.name].append(method)
-            elif tested_methods:
-                self.crawler_process.crawl(spidercls)
+                tested_methods = conman.tested_methods_from_spidercls(spidercls)
+                if opts.list:
+                    for method in tested_methods:
+                        contract_reqs[spidercls.name].append(method)
+                elif tested_methods:
+                    self.crawler_process.crawl(spidercls)
 
         # start checks
         if opts.list:
@@ -95,4 +94,3 @@ class Command(ScrapyCommand):
             result.printErrors()
             result.printSummary(start, stop)
             self.exitcode = int(not result.wasSuccessful())
-

scrapy/commands/crawl.py

@@ -1,7 +1,5 @@
-import os
 from scrapy.commands import ScrapyCommand
-from scrapy.utils.conf import arglist_to_dict
-from scrapy.utils.python import without_none_values
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 from scrapy.exceptions import UsageError
 
 
@@ -19,7 +17,7 @@ class Command(ScrapyCommand):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE",
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE (use - for stdout)")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
@@ -31,21 +29,8 @@ class Command(ScrapyCommand):
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
-            if opts.output == '-':
-                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
-            else:
-                self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(
-                self.settings.getwithbase('FEED_EXPORTERS'))
-            valid_output_formats = feed_exporters.keys()
-            if not opts.output_format:
-                opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
-            if opts.output_format not in valid_output_formats:
-                raise UsageError("Unrecognized output format '%s', set one"
-                                 " using the '-t' switch or as a file extension"
-                                 " from the supported list %s" % (opts.output_format,
-                                                                  tuple(valid_output_formats)))
-            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
+            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) < 1:
@@ -54,8 +39,13 @@ class Command(ScrapyCommand):
             raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
 
-        self.crawler_process.crawl(spname, **opts.spargs)
-        self.crawler_process.start()
+        crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
 
-        if self.crawler_process.bootstrap_failed:
+        if getattr(crawl_defer, 'result', None) is not None and issubclass(crawl_defer.result.type, Exception):
             self.exitcode = 1
+        else:
+            self.crawler_process.start()
+
+            if self.crawler_process.bootstrap_failed or \
+                    (hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
+                self.exitcode = 1

scrapy/commands/fetch.py

@@ -1,5 +1,4 @@
-from __future__ import print_function
-import sys, six
+import sys
 from w3lib.url import is_url
 
 from scrapy.commands import ScrapyCommand
@@ -8,6 +7,7 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 
+
 class Command(ScrapyCommand):
 
     requires_project = False
@@ -24,12 +24,11 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
-        parser.add_option("--spider", dest="spider",
-            help="use this spider")
-        parser.add_option("--headers", dest="headers", action="store_true", \
-            help="print response HTTP headers instead of body")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
-            default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+        parser.add_option("--spider", dest="spider", help="use this spider")
+        parser.add_option("--headers", dest="headers", action="store_true",
+                          help="print response HTTP headers instead of body")
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
+                          default=False, help="do not handle HTTP 3xx status codes and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
@@ -45,8 +44,7 @@ class Command(ScrapyCommand):
             self._print_bytes(response.body)
 
     def _print_bytes(self, bytes_):
-        bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
-        bytes_writer.write(bytes_ + b'\n')
+        sys.stdout.buffer.write(bytes_ + b'\n')
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):

scrapy/commands/genspider.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import os
 import shutil
 import string
@@ -91,8 +90,7 @@ class Command(ScrapyCommand):
             'module': module,
             'name': name,
             'domain': domain,
-            'classname': '%sSpider' % ''.join(s.capitalize() \
-                for s in module.split('_'))
+            'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
         }
         if self.settings.get('NEWSPIDER_MODULE'):
             spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@@ -103,8 +101,8 @@ class Command(ScrapyCommand):
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)
         render_templatefile(spider_file, **tvars)
-        print("Created spider %r using template %r " % (name, \
-            template_name), end=('' if spiders_module else '\n'))
+        print("Created spider %r using template %r "
+              % (name, template_name), end=('' if spiders_module else '\n'))
         if spiders_module:
             print("in module:\n  %s.%s" % (spiders_module.__name__, module))
 

scrapy/commands/list.py

@@ -1,6 +1,6 @@
-from __future__ import print_function
 from scrapy.commands import ScrapyCommand
 
+
 class Command(ScrapyCommand):
 
     requires_project = True

scrapy/commands/parse.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import json
 import logging
 
@@ -51,18 +50,21 @@ class Command(ScrapyCommand):
             help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
+        parser.add_option("--cbkwargs", dest="cbkwargs",
+            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
             help="print each depth level one by one")
 
-
     @property
     def max_level(self):
-        levels = list(self.items.keys()) + list(self.requests.keys())
-        if not levels:
-            return 0
-        return max(levels)
+        max_items, max_requests = 0, 0
+        if self.items:
+            max_items = max(self.items)
+        if self.requests:
+            max_requests = max(self.requests)
+        return max(max_items, max_requests)
 
     def add_items(self, lvl, new_items):
         old_items = self.items.get(lvl, [])
@@ -78,27 +80,26 @@ class Command(ScrapyCommand):
         else:
             items = self.items.get(lvl, [])
 
-        print("# Scraped Items ", "-"*60)
+        print("# Scraped Items ", "-" * 60)
         display.pprint([dict(x) for x in items], colorize=colour)
 
     def print_requests(self, lvl=None, colour=True):
         if lvl is None:
-            levels = list(self.requests.keys())
-            if levels:
-                requests = self.requests[max(levels)]
+            if self.requests:
+                requests = self.requests[max(self.requests)]
             else:
                 requests = []
         else:
             requests = self.requests.get(lvl, [])
 
-        print("# Requests ", "-"*65)
+        print("# Requests ", "-" * 65)
         display.pprint(requests, colorize=colour)
 
     def print_results(self, opts):
         colour = not opts.nocolour
 
         if opts.verbose:
-            for level in range(1, self.max_level+1):
+            for level in range(1, self.max_level + 1):
                 print('\n>>> DEPTH LEVEL: %s <<<' % level)
                 if not opts.noitems:
                     self.print_items(level, colour)
@@ -111,10 +112,11 @@ class Command(ScrapyCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def run_callback(self, response, cb):
+    def run_callback(self, response, callback, cb_kwargs=None):
+        cb_kwargs = cb_kwargs or {}
         items, requests = [], []
 
-        for x in iterate_spider_output(cb(response)):
+        for x in iterate_spider_output(callback(response, **cb_kwargs)):
             if isinstance(x, (BaseItem, dict)):
                 items.append(x)
             elif isinstance(x, Request):
@@ -142,8 +144,7 @@ class Command(ScrapyCommand):
         else:
             self.spidercls = spidercls_for_request(spider_loader, Request(url))
             if not self.spidercls:
-                logger.error('Unable to find spider for: %(url)s',
-                             {'url': url})
+                logger.error('Unable to find spider for: %(url)s', {'url': url})
 
         # Request requires callback argument as callable or None, not string
         request = Request(url, None)
@@ -160,7 +161,7 @@ class Command(ScrapyCommand):
                          {'url': url})
 
     def prepare_request(self, spider, request, opts):
-        def callback(response):
+        def callback(response, **cb_kwargs):
             # memorize first request
             if not self.first_response:
                 self.first_response = response
@@ -175,7 +176,7 @@ class Command(ScrapyCommand):
 
                     if not cb:
                         logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
-                                 {'url': response.url, 'spider': spider.name})
+                                     {'url': response.url, 'spider': spider.name})
                         return
                 else:
                     cb = 'parse'
@@ -192,7 +193,7 @@ class Command(ScrapyCommand):
             # parse items and requests
             depth = response.meta['_depth']
 
-            items, requests = self.run_callback(response, cb)
+            items, requests = self.run_callback(response, cb, cb_kwargs)
             if opts.pipelines:
                 itemproc = self.pcrawler.engine.scraper.itemproc
                 for item in items:
@@ -207,10 +208,14 @@ class Command(ScrapyCommand):
                     req.callback = callback
                 return requests
 
-        #update request meta if any extra meta was passed through the --meta/-m opts.
+        # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
             request.meta.update(opts.meta)
 
+        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
+        if opts.cbkwargs:
+            request.cb_kwargs.update(opts.cbkwargs)
+
         request.meta['_depth'] = 1
         request.meta['_callback'] = request.callback
         request.callback = callback
@@ -221,23 +226,29 @@ class Command(ScrapyCommand):
 
         self.process_spider_arguments(opts)
         self.process_request_meta(opts)
+        self.process_request_cb_kwargs(opts)
 
     def process_spider_arguments(self, opts):
-
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
 
     def process_request_meta(self, opts):
-
         if opts.meta:
             try:
                 opts.meta = json.loads(opts.meta)
             except ValueError:
-                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
-                                "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+                                 "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
 
+    def process_request_cb_kwargs(self, opts):
+        if opts.cbkwargs:
+            try:
+                opts.cbkwargs = json.loads(opts.cbkwargs)
+            except ValueError:
+                raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
+                                 "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
 
     def run(self, args, opts):
         # parse arguments

scrapy/commands/runspider.py

@@ -5,8 +5,7 @@ from importlib import import_module
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.utils.conf import arglist_to_dict
-from scrapy.utils.python import without_none_values
+from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 
 def _import_file(filepath):
@@ -43,7 +42,7 @@ class Command(ScrapyCommand):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
                           help="set spider argument (may be repeated)")
-        parser.add_option("-o", "--output", metavar="FILE",
+        parser.add_option("-o", "--output", metavar="FILE", action="append",
                           help="dump scraped items into FILE (use - for stdout)")
         parser.add_option("-t", "--output-format", metavar="FORMAT",
                           help="format to use for dumping items with -o")
@@ -55,20 +54,8 @@ class Command(ScrapyCommand):
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
         if opts.output:
-            if opts.output == '-':
-                self.settings.set('FEED_URI', 'stdout:', priority='cmdline')
-            else:
-                self.settings.set('FEED_URI', opts.output, priority='cmdline')
-            feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
-            valid_output_formats = feed_exporters.keys()
-            if not opts.output_format:
-                opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
-            if opts.output_format not in valid_output_formats:
-                raise UsageError("Unrecognized output format '%s', set one"
-                                 " using the '-t' switch or as a file extension"
-                                 " from the supported list %s" % (opts.output_format,
-                                                                  tuple(valid_output_formats)))
-            self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
+            feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
+            self.settings.set('FEEDS', feeds, priority='cmdline')
 
     def run(self, args, opts):
         if len(args) != 1:

scrapy/commands/settings.py

@@ -1,9 +1,9 @@
-from __future__ import print_function
 import json
 
 from scrapy.commands import ScrapyCommand
 from scrapy.settings import BaseSettings
 
+
 class Command(ScrapyCommand):
 
     requires_project = False

scrapy/commands/shell.py

@@ -6,8 +6,8 @@ See documentation in docs/topics/shell.rst
 from threading import Thread
 
 from scrapy.commands import ScrapyCommand
-from scrapy.shell import Shell
 from scrapy.http import Request
+from scrapy.shell import Shell
 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
 from scrapy.utils.url import guess_scheme
 
@@ -28,7 +28,8 @@ class Command(ScrapyCommand):
         return "Interactive scraping console"
 
     def long_desc(self):
-        return "Interactive console for scraping the given url"
+        return ("Interactive console for scraping the given url or file. "
+                "Use ./file.html syntax or full path for local file.")
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
@@ -36,7 +37,7 @@ class Command(ScrapyCommand):
             help="evaluate the code in the shell, print the result and exit")
         parser.add_option("--spider", dest="spider",
             help="use this spider")
-        parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
+        parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
             default=False, help="do not handle HTTP 3xx status codes and print response as-is")
 
     def update_vars(self, vars):

scrapy/commands/startproject.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import re
 import os
 import string
@@ -44,8 +43,8 @@ class Command(ScrapyCommand):
                 return False
 
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):
-            print('Error: Project names must begin with a letter and contain'\
-                    ' only\nletters, numbers and underscores')
+            print('Error: Project names must begin with a letter and contain'
+                  ' only\nletters, numbers and underscores')
         elif _module_exists(project_name):
             print('Error: Module %r already exists' % project_name)
         else:
@@ -119,4 +118,3 @@ class Command(ScrapyCommand):
         _templates_base_dir = self.settings['TEMPLATES_DIR'] or \
             join(scrapy.__path__[0], 'templates')
         return join(_templates_base_dir, 'project')
-

scrapy/commands/version.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.utils.versions import scrapy_components_versions
@@ -30,4 +28,3 @@ class Command(ScrapyCommand):
                 print(patt % (name, version))
         else:
             print("Scrapy %s" % scrapy.__version__)
-

scrapy/commands/view.py

@@ -1,6 +1,7 @@
-from scrapy.commands import fetch, ScrapyCommand
+from scrapy.commands import fetch
 from scrapy.utils.response import open_in_browser
 
+
 class Command(fetch.Command):
 
     def short_desc(self):

scrapy/conf.py

@@ -1,13 +0,0 @@
-# This module is kept for backwards compatibility, so users can import
-# scrapy.conf.settings and get the settings they expect
-
-import sys
-
-if 'scrapy.cmdline' not in sys.modules:
-    from scrapy.utils.project import get_project_settings
-    settings = get_project_settings()
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.conf` is deprecated, use `crawler.settings` attribute instead",
-    ScrapyDeprecationWarning, stacklevel=2)

scrapy/contracts/__init__.py

@@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
 
 
-class ContractsManager(object):
+class ContractsManager:
     contracts = {}
 
     def __init__(self, contracts):
@@ -90,11 +90,11 @@ class ContractsManager(object):
         cb = request.callback
 
         @wraps(cb)
-        def cb_wrapper(response):
+        def cb_wrapper(response, **cb_kwargs):
             try:
-                output = cb(response)
+                output = cb(response, **cb_kwargs)
                 output = list(iterate_spider_output(output))
-            except:
+            except Exception:
                 case = _create_testcase(method, 'callback')
                 results.addError(case, sys.exc_info())
 
@@ -107,7 +107,7 @@ class ContractsManager(object):
         request.errback = eb_wrapper
 
 
-class Contract(object):
+class Contract:
     """ Abstract class for contracts """
     request_cls = None
 
@@ -121,7 +121,7 @@ class Contract(object):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
+            def wrapper(response, **cb_kwargs):
                 try:
                     results.startTest(self.testcase_pre)
                     self.pre_process(response)
@@ -133,7 +133,7 @@ class Contract(object):
                 else:
                     results.addSuccess(self.testcase_pre)
                 finally:
-                    return list(iterate_spider_output(cb(response)))
+                    return list(iterate_spider_output(cb(response, **cb_kwargs)))
 
             request.callback = wrapper
 
@@ -144,8 +144,8 @@ class Contract(object):
             cb = request.callback
 
             @wraps(cb)
-            def wrapper(response):
-                output = list(iterate_spider_output(cb(response)))
+            def wrapper(response, **cb_kwargs):
+                output = list(iterate_spider_output(cb(response, **cb_kwargs)))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)