Merge remote-tracking branch 'upstream/master' into docs_crawlspider_link_text

.coveragerc

@@ -4,5 +4,3 @@ include = scrapy/*
 omit =
   tests/*
   scrapy/xlib/*
-  scrapy/conf.py
-  scrapy/log.py

appveyor.yml

@@ -12,7 +12,8 @@ branches:
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE"
+  - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
+  - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
   - "pip install -U tox"
 
 build: false

conftest.py

@@ -9,10 +9,6 @@ def _py_files(folder):
 
 
 collect_ignore = [
-    # deprecated or moved modules
-    "scrapy/conf.py",
-    "scrapy/log.py",
-
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
 

docs/Makefile

@@ -82,6 +82,9 @@ pydoc-topics: build
 	@echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \
 	      "into the Lib/ directory"
 
+coverage: BUILDER = coverage
+coverage: build
+
 htmlview: html
 	 $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \
 	 os.path.realpath('build/html/index.html'))"

docs/conf.py

@@ -28,7 +28,8 @@ sys.path.insert(0, path.dirname(path.dirname(__file__)))
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
     'scrapydocs',
-    'sphinx.ext.autodoc'
+    'sphinx.ext.autodoc',
+    'sphinx.ext.coverage',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -218,3 +219,31 @@ linkcheck_ignore = [
     'http://localhost:\d+', 'http://hg.scrapy.org',
     'http://directory.google.com/'
 ]
+
+
+# Options for the Coverage extension
+# ----------------------------------
+coverage_ignore_pyobjects = [
+    # Contract’s add_pre_hook and add_post_hook are not documented because
+    # they should be transparent to contract developers, for whom pre_hook and
+    # post_hook should be the actual concern.
+    r'\bContract\.add_(pre|post)_hook$',
+
+    # ContractsManager is an internal class, developers are not expected to
+    # interact with it directly in any way.
+    r'\bContractsManager\b$',
+
+    # For default contracts we only want to document their general purpose in
+    # their constructor, the methods they reimplement to achieve that purpose
+    # should be irrelevant to developers using those contracts.
+    r'\w+Contract\.(adjust_request_args|(pre|post)_process)$',
+
+    # Methods of downloader middlewares are not documented, only the classes
+    # themselves, since downloader middlewares are controlled through Scrapy
+    # settings.
+    r'^scrapy\.downloadermiddlewares\.\w*?\.(\w*?Middleware|DownloaderStats)\.',
+
+    # Base classes of downloader middlewares are implementation details that
+    # are not meant for users.
+    r'^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware',
+]

docs/contributing.rst

@@ -99,6 +99,15 @@ Well-written patches should:
   the documentation changes in the same patch.  See `Documentation policies`_
   below.
 
+* if you're adding a private API, please add a regular expression to the
+  ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new
+  private API from documentation coverage checks.
+
+  To see if your private API is skipped properly, generate a documentation
+  coverage report as follows::
+
+      tox -e docs-coverage
+
 .. _submitting-patches:
 
 Submitting patches
@@ -167,8 +176,9 @@ Documentation policies
 
 For reference documentation of API members (classes, methods, etc.) use
 docstrings and make sure that the Sphinx documentation uses the autodoc_
-extension to pull the docstrings. API reference documentation should be
-IDE-friendly: short, to the point, and it may provide short examples.
+extension to pull the docstrings. API reference documentation should follow
+docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point,
+and it may provide short examples.
 
 Other types of documentation, such as tutorials or topics, should be covered in
 files within the ``docs/`` directory. This includes documentation that is
@@ -205,6 +215,29 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
     ``tox -- tests/test_loader.py``
 
+To run the tests on a specific tox_ environment, use ``-e <name>`` with an
+environment name from ``tox.ini``. For example, to run the tests with Python
+3.6 use::
+
+    tox -e py36
+
+You can also specify a comma-separated list of environmets, and use `tox’s
+parallel mode`_ to run the tests on multiple environments in parallel::
+
+    tox -e py27,py36 -p auto
+
+To pass command-line options to pytest_, add them after ``--`` in your call to
+tox_. Using ``--`` overrides the default positional arguments defined in
+``tox.ini``, so you must include those default positional arguments
+(``scrapy tests``) after ``--`` as well::
+
+    tox -- scrapy tests -x  # stop after first failure
+
+You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
+the Python 3.6 tox_ environment using all your CPU cores::
+
+    tox -e py36 -- scrapy tests -n auto
+
 To see coverage report install `coverage`_ (``pip install coverage``) and run:
 
     ``coverage report``
@@ -237,5 +270,9 @@ And their unit-tests are in::
 .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
 .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
 .. _open issues: https://github.com/scrapy/scrapy/issues
-.. _pull request: https://help.github.com/send-pull-requests/
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
+.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _pytest: https://docs.pytest.org/en/latest/usage.html
+.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
 .. _tox: https://pypi.python.org/pypi/tox
+.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode

docs/faq.rst

@@ -129,13 +129,23 @@ Does Scrapy crawl in breadth-first or depth-first order?
 
 By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which
 basically means that it crawls in `DFO order`_. This order is more convenient
-in most cases. If you do want to crawl in true `BFO order`_, you can do it by
+in most cases.
+
+If you do want to crawl in true `BFO order`_, you can do it by
 setting the following settings::
 
     DEPTH_PRIORITY = 1
     SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue'
     SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue'
 
+While pending requests are below the configured values of
+:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
+concurrently. As a result, the first few requests of a crawl rarely follow the
+desired order. Lowering those settings to ``1`` enforces the desired order, but
+it significantly slows down the crawl as a whole.
+
+
 My Scrapy crawler has memory leaks. What can I do?
 --------------------------------------------------
 
@@ -319,6 +329,29 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
+How to split an item into multiple items in an item pipeline?
+-------------------------------------------------------------
+
+:ref:`Item pipelines <topics-item-pipeline>` cannot yield multiple items per
+input item. :ref:`Create a spider middleware <custom-spider-middleware>`
+instead, and use its
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+method for this puspose. For example::
+
+    from copy import deepcopy
+
+    from scrapy.item import BaseItem
+
+
+    class MultiplyItemsMiddleware:
+
+        def process_spider_output(self, response, result, spider):
+            for item in result:
+                if isinstance(item, (BaseItem, dict)):
+                    for _ in range(item['multiply_by']):
+                        yield deepcopy(item)
+
+
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)
 .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search

docs/index.rst

@@ -158,6 +158,7 @@ Solving specific problems
    topics/practices
    topics/broad-crawls
    topics/developer-tools
+   topics/dynamic-content
    topics/leaks
    topics/media-pipeline
    topics/deploy
@@ -183,6 +184,9 @@ Solving specific problems
 :doc:`topics/developer-tools`
     Learn how to scrape with your browser's developer tools.
 
+:doc:`topics/dynamic-content`
+    Read webpage data that is loaded dynamically.
+
 :doc:`topics/leaks`
     Learn how to find and get rid of memory leaks in your crawler.
 

docs/intro/tutorial.rst

@@ -205,7 +205,7 @@ Extracting data
 ---------------
 
 The best way to learn how to extract data with Scrapy is trying selectors
-using the shell :ref:`Scrapy shell <topics-shell>`. Run::
+using the :ref:`Scrapy shell <topics-shell>`. Run::
 
     scrapy shell 'http://quotes.toscrape.com/page/1/'
 
@@ -296,8 +296,8 @@ expressions`_::
 
 In order to find the proper CSS selectors to use, you might find useful opening
 the response page from the shell in your web browser using ``view(response)``.
-You can use your browser developer tools to inspect the HTML and come up
-with a selector (see section about :ref:`topics-developer-tools`).
+You can use your browser's developer tools to inspect the HTML and come up
+with a selector (see :ref:`topics-developer-tools`).
 
 `Selector Gadget`_ is also a nice tool to quickly find CSS selector for
 visually selected elements, which works in many browsers.
@@ -379,11 +379,11 @@ variable, so that we can run our CSS selectors directly on a particular quote::
 
     >>> quote = response.css("div.quote")[0]
 
-Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
+Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
 using the ``quote`` object we just created::
 
-    >>> title = quote.css("span.text::text").get()
-    >>> title
+    >>> text = quote.css("span.text::text").get()
+    >>> text
     '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
     >>> author = quote.css("small.author::text").get()
     >>> author
@@ -511,7 +511,7 @@ We can try extracting it in the shell::
     '<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
 
 This gets the anchor element, but we want the attribute ``href``. For that,
-Scrapy supports a CSS extension that let's you select the attribute contents,
+Scrapy supports a CSS extension that lets you select the attribute contents,
 like this::
 
     >>> response.css('li.next a::attr(href)').get()

docs/requirements.txt

@@ -1,2 +1,2 @@
-Sphinx>=1.6
+Sphinx>=2.1
 sphinx_rtd_theme

docs/topics/api.rst

@@ -99,6 +99,8 @@ how you :ref:`configure the downloader middlewares
 
         Returns a deferred that is fired when the crawl is finished.
 
+    .. automethod:: stop
+
 .. autoclass:: CrawlerRunner
    :members:
 
@@ -154,7 +156,7 @@ Settings API
 SpiderLoader API
 ================
 
-.. module:: scrapy.loader
+.. module:: scrapy.spiderloader
    :synopsis: The spider loader
 
 .. class:: SpiderLoader

docs/topics/broad-crawls.rst

@@ -39,6 +39,17 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with
 concrete suggestions of Scrapy settings to tune in order to achieve an
 efficient broad crawl.
 
+Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
+=================================================
+
+Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
+It works best during single-domain crawl. It does not work well with crawling
+many different domains in parallel
+
+To apply the recommended priority queue use::
+
+    SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+
 Increase concurrency
 ====================
 
@@ -85,7 +96,7 @@ When doing broad crawls you are often only interested in the crawl rates you
 get and any errors found. These stats are reported by Scrapy when using the
 ``INFO`` log level. In order to save CPU (and log storage requirements) you
 should not use ``DEBUG`` log level when preforming large broad crawls in
-production. Using ``DEBUG`` level when developing your (broad) crawler may be 
+production. Using ``DEBUG`` level when developing your (broad) crawler may be
 fine though.
 
 To set the log level use::

docs/topics/commands.rst

@@ -461,6 +461,9 @@ Supported options:
 * ``--meta`` or ``-m``: additional request meta that will be passed to the callback 
   request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
 
+* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
+  This must be a valid json string. Example: --cbkwargs='{"foo" : "bar"}'
+
 * ``--pipelines``: process items through pipelines
 
 * ``--rules`` or ``-r``: use :class:`~scrapy.spiders.CrawlSpider`

docs/topics/contracts.rst

@@ -120,3 +120,23 @@ get the failures pretty printed::
             for header in self.args:
                 if header not in response.headers:
                     raise ContractFail('X-CustomHeader not present')
+
+
+Detecting check runs
+====================
+
+When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
+set to the ``true`` string. You can use `os.environ`_ to perform any change to
+your spiders or your settings when ``scrapy check`` is used::
+
+    import os
+    import scrapy
+
+    class ExampleSpider(scrapy.Spider):
+        name = 'example'
+
+        def __init__(self):
+            if os.environ.get('SCRAPY_CHECK'):
+                pass  # Do some scraper adjustments when a check is running
+
+.. _os.environ: https://docs.python.org/3/library/os.html#os.environ

docs/topics/debug.rst

@@ -28,16 +28,15 @@ Consider the following scrapy spider below::
             item = MyItem()
             # populate `item` fields
             # and extract item_details_url
-            yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})
+            yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item})
 
-        def parse_details(self, response):
-            item = response.meta['item']
+        def parse_details(self, response, item):
             # populate more `item` fields
             return item
 
 Basically this is a simple spider which parses two pages of items (the
 start_urls). Items also have a details page with additional information, so we
-use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a
+use the ``cb_kwargs`` functionality of :class:`~scrapy.http.Request` to pass a
 partially populated item.
 
 
@@ -100,8 +99,7 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see
 
     from scrapy.shell import inspect_response
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item
@@ -134,8 +132,7 @@ Logging is another useful option for getting information about your spider run.
 Although not as convenient, it comes with the advantage that the logs will be
 available in all future runs should they be necessary again::
 
-    def parse_details(self, response):
-        item = response.meta.get('item', None)
+    def parse_details(self, response, item=None):
         if item:
             # populate more `item` fields
             return item

docs/topics/downloader-middleware.rst

@@ -349,7 +349,7 @@ HttpCacheMiddleware
         * :ref:`httpcache-storage-leveldb`
 
     You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
-    setting. Or you can also implement your own storage backend.
+    setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
 
     Scrapy ships with two HTTP cache policies:
 
@@ -496,6 +496,61 @@ In order to use this storage backend:
 .. _LevelDB: https://github.com/google/leveldb
 .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
 
+.. _httpcache-storage-custom:
+
+Writing your own storage backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can implement a cache storage backend by creating a Python class that
+defines the methods described below.
+
+.. module:: scrapy.extensions.httpcache
+
+.. class:: CacheStorage
+
+    .. method:: open_spider(spider)
+
+      This method gets called after a spider has been opened for crawling. It handles 
+      the :signal:`open_spider <spider_opened>` signal.
+
+      :param spider: the spider which has been opened
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+    .. method:: close_spider(spider)
+
+      This method gets called after a spider has been closed. It handles 
+      the :signal:`close_spider <spider_closed>` signal. 
+
+      :param spider: the spider which has been closed
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+    .. method:: retrieve_response(spider, request)
+
+      Return response if present in cache, or ``None`` otherwise.
+
+      :param spider: the spider which generated the request
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+      :param request: the request to find cached reponse for
+      :type request: :class:`~scrapy.http.Request` object
+
+    .. method:: store_response(spider, request, response)
+
+      Store the given response in the cache.
+
+      :param spider: the spider for which the response is intended
+      :type spider: :class:`~scrapy.spiders.Spider` object
+
+      :param request: the corresponding request the spider generated
+      :type request: :class:`~scrapy.http.Request` object
+
+      :param response: the response to store in the cache
+      :type response: :class:`~scrapy.http.Response` object
+
+In order to use your storage backend, set:
+
+* :setting:`HTTPCACHE_STORAGE` to the Python import path of your custom storage class.
+
 
 HTTPCache middleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -805,6 +860,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
 * :setting:`METAREFRESH_ENABLED`
+* :setting:`METAREFRESH_IGNORE_TAGS`
 * :setting:`METAREFRESH_MAXDELAY`
 
 This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`,
@@ -826,6 +882,15 @@ Default: ``True``
 
 Whether the Meta Refresh middleware will be enabled.
 
+.. setting:: METAREFRESH_IGNORE_TAGS
+
+METAREFRESH_IGNORE_TAGS
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``['script', 'noscript']``
+
+Meta tags within these tags are ignored.
+
 .. setting:: METAREFRESH_MAXDELAY
 
 METAREFRESH_MAXDELAY

docs/topics/dynamic-content.rst

@@ -0,0 +1,246 @@
+.. _topics-dynamic-content:
+
+====================================
+Selecting dynamically-loaded content
+====================================
+
+Some webpages show the desired data when you load them in a web browser.
+However, when you download them using Scrapy, you cannot reach the desired data
+using :ref:`selectors <topics-selectors>`.
+
+When this happens, the recommended approach is to
+:ref:`find the data source <topics-finding-data-source>` and extract the data
+from it.
+
+If you fail to do that, and you can nonetheless access the desired data through
+the :ref:`DOM <topics-livedom>` from your web browser, see
+:ref:`topics-javascript-rendering`.
+
+.. _topics-finding-data-source:
+
+Finding the data source
+=======================
+
+To extract the desired data, you must first find its source location.
+
+If the data is in a non-text-based format, such as an image or a PDF document,
+use the :ref:`network tool <topics-network-tool>` of your web browser to find
+the corresponding request, and :ref:`reproduce it
+<topics-reproducing-requests>`.
+
+If your web browser lets you select the desired data as text, the data may be
+defined in embedded JavaScript code, or loaded from an external resource in a
+text-based format.
+
+In that case, you can use a tool like wgrep_ to find the URL of that resource.
+
+If the data turns out to come from the original URL itself, you must
+:ref:`inspect the source code of the webpage <topics-inspecting-source>` to
+determine where the data is located.
+
+If the data comes from a different URL, you will need to :ref:`reproduce the
+corresponding request <topics-reproducing-requests>`.
+
+.. _topics-inspecting-source:
+
+Inspecting the source code of a webpage
+=======================================
+
+Sometimes you need to inspect the source code of a webpage (not the
+:ref:`DOM <topics-livedom>`) to determine where some desired data is located.
+
+Use Scrapy’s :command:`fetch` command to download the webpage contents as seen
+by Scrapy::
+
+    scrapy fetch --nolog https://example.com > response.html
+
+If the desired data is in embedded JavaScript code within a ``<script/>``
+element, see :ref:`topics-parsing-javascript`.
+
+If you cannot find the desired data, first make sure it’s not just Scrapy:
+download the webpage with an HTTP client like curl_ or wget_ and see if the
+information can be found in the response they get.
+
+If they get a response with the desired data, modify your Scrapy
+:class:`~scrapy.http.Request` to match that of the other HTTP client. For
+example, try using the same user-agent string (:setting:`USER_AGENT`) or the
+same :attr:`~scrapy.http.Request.headers`.
+
+If they also get a response without the desired data, you’ll need to take
+steps to make your request more similar to that of the web browser. See
+:ref:`topics-reproducing-requests`.
+
+.. _topics-reproducing-requests:
+
+Reproducing requests
+====================
+
+Sometimes we need to reproduce a request the way our web browser performs it.
+
+Use the :ref:`network tool <topics-network-tool>` of your web browser to see
+how your web browser performs the desired request, and try to reproduce that
+request with Scrapy.
+
+It might be enough to yield a :class:`~scrapy.http.Request` with the same HTTP
+method and URL. However, you may also need to reproduce the body, headers and
+form parameters (see :class:`~scrapy.http.FormRequest`) of that request.
+
+Once you get the expected response, you can :ref:`extract the desired data from
+it <topics-handling-response-formats>`.
+
+You can reproduce any request with Scrapy. However, some times reproducing all
+necessary requests may not seem efficient in developer time. If that is your
+case, and crawling speed is not a major concern for you, you can alternatively
+consider :ref:`JavaScript pre-rendering <topics-javascript-rendering>`.
+
+If you get the expected response `sometimes`, but not always, the issue is
+probably not your request, but the target server. The target server might be
+buggy, overloaded, or :ref:`banning <bans>` some of your requests.
+
+.. _topics-handling-response-formats:
+
+Handling different response formats
+===================================
+
+Once you have a response with the desired data, how you extract the desired
+data from it depends on the type of response:
+
+-   If the response is HTML or XML, use :ref:`selectors
+    <topics-selectors>` as usual.
+
+-   If the response is JSON, use `json.loads`_ to load the desired data from
+    :attr:`response.text <scrapy.http.TextResponse.text>`::
+
+        data = json.loads(response.text)
+
+    If the desired data is inside HTML or XML code embedded within JSON data,
+    you can load that HTML or XML code into a
+    :class:`~scrapy.selector.Selector` and then
+    :ref:`use it <topics-selectors>` as usual::
+
+        selector = Selector(data['html'])
+
+-   If the response is JavaScript, or HTML with a ``<script/>`` element
+    containing the desired data, see :ref:`topics-parsing-javascript`.
+
+-   If the response is CSS, use a `regular expression`_ to extract the desired
+    data from :attr:`response.text <scrapy.http.TextResponse.text>`.
+
+.. _topics-parsing-images:
+
+-   If the response is an image or another format based on images (e.g. PDF),
+    read the response as bytes from
+    :attr:`response.body <scrapy.http.TextResponse.body>` and use an OCR
+    solution to extract the desired data as text.
+
+    For example, you can use pytesseract_. To read a table from a PDF,
+    `tabula-py`_ may be a better choice.
+
+-   If the response is SVG, or HTML with embedded SVG containing the desired
+    data, you may be able to extract the desired data using
+    :ref:`selectors <topics-selectors>`, since SVG is based on XML.
+
+    Otherwise, you might need to convert the SVG code into a raster image, and
+    :ref:`handle that raster image <topics-parsing-images>`.
+
+.. _topics-parsing-javascript:
+
+Parsing JavaScript code
+=======================
+
+If the desired data is hardcoded in JavaScript, you first need to get the
+JavaScript code:
+
+-   If the JavaScript code is in a JavaScript file, simply read
+    :attr:`response.text <scrapy.http.TextResponse.text>`.
+
+-   If the JavaScript code is within a ``<script/>`` element of an HTML page,
+    use :ref:`selectors <topics-selectors>` to extract the text within that
+    ``<script/>`` element.
+
+Once you have a string with the JavaScript code, you can extract the desired
+data from it:
+
+-   You might be able to use a `regular expression`_ to extract the desired
+    data in JSON format, which you can then parse with `json.loads`_.
+
+    For example, if the JavaScript code contains a separate line like
+    ``var data = {"field": "value"};`` you can extract that data as follows::
+
+        >>> pattern = r'\bvar\s+data\s*=\s*(\{.*?\})\s*;\s*\n'
+        >>> json_data = response.css('script::text').re_first(pattern)
+        >>> json.loads(json_data)
+        {'field': 'value'}
+
+-   Otherwise, use js2xml_ to convert the JavaScript code into an XML document
+    that you can parse using :ref:`selectors <topics-selectors>`.
+
+    For example, if the JavaScript code contains
+    ``var data = {field: "value"};`` you can extract that data as follows::
+
+        >>> import js2xml
+        >>> import lxml.etree
+        >>> from parsel import Selector
+        >>> javascript = response.css('script::text').get()
+        >>> xml = lxml.etree.tostring(js2xml.parse(javascript), encoding='unicode')
+        >>> selector = Selector(text=xml)
+        >>> selector.css('var[name="data"]').get()
+        '<var name="data"><object><property name="field"><string>value</string></property></object></var>'
+
+.. _topics-javascript-rendering:
+
+Pre-rendering JavaScript
+========================
+
+On webpages that fetch data from additional requests, reproducing those
+requests that contain the desired data is the preferred approach. The effort is
+often worth the result: structured, complete data with minimum parsing time and
+network transfer.
+
+However, sometimes it can be really hard to reproduce certain requests. Or you
+may need something that no request can give you, such as a screenshot of a
+webpage as seen in a web browser.
+
+In these cases use the Splash_ JavaScript-rendering service, along with
+`scrapy-splash`_ for seamless integration.
+
+Splash returns as HTML the :ref:`DOM <topics-livedom>` of a webpage, so that
+you can parse it with :ref:`selectors <topics-selectors>`. It provides great
+flexibility through configuration_ or scripting_.
+
+If you need something beyond what Splash offers, such as interacting with the
+DOM on-the-fly from Python code instead of using a previously-written script,
+or handling multiple web browser windows, you might need to
+:ref:`use a headless browser <topics-headless-browsing>` instead.
+
+.. _configuration: https://splash.readthedocs.io/en/stable/api.html
+.. _scripting: https://splash.readthedocs.io/en/stable/scripting-tutorial.html
+
+.. _topics-headless-browsing:
+
+Using a headless browser
+========================
+
+A `headless browser`_ is a special web browser that provides an API for
+automation.
+
+The easiest way to use a headless browser with Scrapy is to use Selenium_,
+along with `scrapy-selenium`_ for seamless integration.
+
+
+.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
+.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
+.. _curl: https://curl.haxx.se/
+.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
+.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
+.. _js2xml: https://github.com/scrapinghub/js2xml
+.. _json.loads: https://docs.python.org/library/json.html#json.loads
+.. _pytesseract: https://github.com/madmaze/pytesseract
+.. _regular expression: https://docs.python.org/library/re.html
+.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
+.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
+.. _Selenium: https://www.seleniumhq.org/
+.. _Splash: https://github.com/scrapinghub/splash
+.. _tabula-py: https://github.com/chezou/tabula-py
+.. _wget: https://www.gnu.org/software/wget/
+.. _wgrep: https://github.com/stav/wgrep

docs/topics/jobs.rst

@@ -81,7 +81,8 @@ So, for example, this won't work::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
+        return scrapy.Request('http://www.example.com',
+                              callback=lambda r: self.other_callback(r, somearg))
 
     def other_callback(self, response, somearg):
         print("the argument passed is: %s" % somearg)
@@ -90,10 +91,10 @@ But this will::
 
     def some_callback(self, response):
         somearg = 'test'
-        return scrapy.Request('http://www.example.com', callback=self.other_callback, meta={'somearg': somearg})
+        return scrapy.Request('http://www.example.com',
+                              callback=self.other_callback, cb_kwargs={'somearg': somearg})
 
-    def other_callback(self, response):
-        somearg = response.meta['somearg']
+    def other_callback(self, response, somearg):
         print("the argument passed is: %s" % somearg)
 
 If you wish to log the requests that couldn't be serialized, you can set the

docs/topics/leaks.rst

@@ -27,10 +27,11 @@ Common causes of memory leaks
 
 It happens quite often (sometimes by accident, sometimes on purpose) that the
 Scrapy developer passes objects referenced in Requests (for example, using the
-:attr:`~scrapy.http.Request.meta` attribute or the request callback function)
-and that effectively bounds the lifetime of those referenced objects to the
-lifetime of the Request. This is, by far, the most common cause of memory leaks
-in Scrapy projects, and a quite difficult one to debug for newcomers.
+:attr:`~scrapy.http.Request.cb_kwargs` or :attr:`~scrapy.http.Request.meta`
+attributes or the request callback function) and that effectively bounds the
+lifetime of those referenced objects to the lifetime of the Request. This is,
+by far, the most common cause of memory leaks in Scrapy projects, and a quite
+difficult one to debug for newcomers.
 
 In big projects, the spiders are typically written by different people and some
 of those spiders could be "leaking" and thus affecting the rest of the other
@@ -48,7 +49,8 @@ Too Many Requests?
 
 By default Scrapy keeps the request queue in memory; it includes
 :class:`~scrapy.http.Request` objects and all objects
-referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.meta`).
+referenced in Request attributes (e.g. in :attr:`~scrapy.http.Request.cb_kwargs`
+and :attr:`~scrapy.http.Request.meta`).
 While not necessarily a leak, this can take a lot of memory. Enabling
 :ref:`persistent job queue <topics-jobs>` could help keeping memory usage
 in control.
@@ -101,7 +103,7 @@ Let's see a concrete example of a hypothetical case of memory leaks.
 Suppose we have some spider with a line similar to this one::
 
     return Request("http://www.somenastyspider.com/product.php?pid=%d" % product_id,
-        callback=self.parse, meta={referer: response})
+                   callback=self.parse, cb_kwargs={'referer': response})
 
 That line is passing a response reference inside a request which effectively
 ties the response lifetime to the requests' one, and that would definitely

docs/topics/logging.rst

@@ -238,9 +238,10 @@ scrapy.utils.log module
 
 .. autofunction:: configure_logging
 
-    ``configure_logging`` is automatically called when using Scrapy commands,
-    but needs to be called explicitly when running custom scripts. In that
-    case, its usage is not required but it's recommended.
+    ``configure_logging`` is automatically called when using Scrapy commands
+    or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly
+    when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
+    In that case, its usage is not required but it's recommended.
 
     If you plan on configuring the handlers yourself is still recommended you
     call this function, passing ``install_root_handler=False``. Bear in mind

docs/topics/media-pipeline.rst

@@ -392,6 +392,36 @@ See here the methods that you can override in your custom Files Pipeline:
 
 .. class:: FilesPipeline
 
+   .. method:: file_path(request, response, info)
+
+      This method is called once per downloaded item. It returns the
+      download path of the file originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>` and
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+
+      You can override this method to customize the download path of each file.
+
+      For example, if file URLs end like regular paths (e.g.
+      ``https://example.com/a/b/c/foo.png``), you can use the following
+      approach to download all files into the ``files`` folder with their
+      original filenames (e.g. ``files/foo.png``)::
+
+        import os
+        from urllib.parse import urlparse
+
+        from scrapy.pipelines.files import FilesPipeline
+
+        class MyFilesPipeline(FilesPipeline):
+
+            def file_path(self, request, response, info):
+                return 'files/' + os.path.basename(urlparse(request.url).path)
+
+      By default the :meth:`file_path` method returns
+      ``full/<request URL hash>.<extension>``.
+
    .. method:: FilesPipeline.get_media_requests(item, info)
 
       As seen on the workflow, the pipeline will get the URLs of the images to
@@ -475,6 +505,36 @@ See here the methods that you can override in your custom Images Pipeline:
     The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
     customizing the field names and adding custom behavior for images.
 
+   .. method:: file_path(request, response, info)
+
+      This method is called once per downloaded item. It returns the
+      download path of the file originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>` and
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
+
+      You can override this method to customize the download path of each file.
+
+      For example, if file URLs end like regular paths (e.g.
+      ``https://example.com/a/b/c/foo.png``), you can use the following
+      approach to download all files into the ``files`` folder with their
+      original filenames (e.g. ``files/foo.png``)::
+
+        import os
+        from urllib.parse import urlparse
+
+        from scrapy.pipelines.images import ImagesPipeline
+
+        class MyImagesPipeline(ImagesPipeline):
+
+            def file_path(self, request, response, info):
+                return 'files/' + os.path.basename(urlparse(request.url).path)
+
+      By default the :meth:`file_path` method returns
+      ``full/<request URL hash>.<extension>``.
+
    .. method:: ImagesPipeline.get_media_requests(item, info)
 
       Works the same way as :meth:`FilesPipeline.get_media_requests` method,

docs/topics/request-response.rst

@@ -24,7 +24,7 @@ below in :ref:`topics-request-response-ref-request-subclasses` and
 Request objects
 ===============
 
-.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags])
+.. class:: Request(url[, callback, method='GET', headers, body, cookies, meta, encoding='utf-8', priority=0, dont_filter=False, errback, flags, cb_kwargs])
 
     A :class:`Request` object represents an HTTP request, which is usually
     generated in the Spider and executed by the Downloader, and thus generating
@@ -126,6 +126,9 @@ Request objects
     :param flags:  Flags sent to the request, can be used for logging or similar purposes.
     :type flags: list
 
+    :param cb_kwargs: A dict with arbitrary data that will be passed as keyword arguments to the Request's callback.
+    :type cb_kwargs: dict
+
     .. attribute:: Request.url
 
         A string containing the URL of this request. Keep in mind that this
@@ -165,6 +168,17 @@ Request objects
         ``copy()`` or ``replace()`` methods, and can also be accessed, in your
         spider, from the ``response.meta`` attribute.
 
+    .. attribute:: Request.cb_kwargs
+
+        A dictionary that contains arbitrary metadata for this request. Its contents
+        will be passed to the Request's callback as keyword arguments. It is empty
+        for new Requests, which means by default callbacks only get a :class:`Response`
+        object as argument.
+
+        This dict is `shallow copied`_ when the request is cloned using the
+        ``copy()`` or ``replace()`` methods, and can also be accessed, in your
+        spider, from the ``response.cb_kwargs`` attribute.
+
     .. _shallow copied: https://docs.python.org/2/library/copy.html
 
     .. method:: Request.copy()
@@ -172,12 +186,12 @@ Request objects
        Return a new Request which is a copy of this Request. See also:
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
-    .. method:: Request.replace([url, method, headers, body, cookies, meta, encoding, dont_filter, callback, errback])
+    .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
 
        Return a Request object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
-       attribute :attr:`Request.meta` is copied by default (unless a new value
-       is given in the ``meta`` argument). See also
+       :attr:`Request.cb_kwargs` and :attr:`Request.meta` attributes are shallow
+       copied by default (unless new values are given as arguments). See also
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
 .. _topics-request-response-ref-request-callback-arguments:
@@ -200,25 +214,31 @@ Example::
         self.logger.info("Visited %s", response.url)
 
 In some cases you may be interested in passing arguments to those callback
-functions so you can receive the arguments later, in the second callback. You
-can use the :attr:`Request.meta` attribute for that.
+functions so you can receive the arguments later, in the second callback.
+The following example shows how to achieve this by using the
+:attr:`Request.cb_kwargs` attribute:
 
-Here's an example of how to pass an item using this mechanism, to populate
-different fields from different pages::
+::
 
-    def parse_page1(self, response):
-        item = MyItem()
-        item['main_url'] = response.url
-        request = scrapy.Request("http://www.example.com/some_page.html",
-                                 callback=self.parse_page2)
-        request.meta['item'] = item
+    def parse(self, response):
+        request = scrapy.Request('http://www.example.com/index.html',
+                                 callback=self.parse_page2,
+                                 cb_kwargs=dict(main_url=response.url))
+        request.cb_kwargs['foo'] = 'bar'  # add more arguments for the callback
         yield request
 
-    def parse_page2(self, response):
-        item = response.meta['item']
-        item['other_url'] = response.url
-        yield item
+    def parse_page2(self, response, main_url, foo):
+        yield dict(
+            main_url=main_url,
+            other_url=response.url,
+            foo=foo,
+        )
 
+.. caution:: :attr:`Request.cb_kwargs` was introduced in version ``1.7``.
+   Prior to that, using :attr:`Request.meta` was recommended for passing
+   information around callbacks. After ``1.7``, :attr:`Request.cb_kwargs`
+   became the preferred way for handling user information, leaving :attr:`Request.meta`
+   for communication with components like middlewares and extensions.
 
 .. _topics-request-response-ref-errbacks:
 

docs/topics/settings.rst

@@ -897,6 +897,16 @@ Default: ``False``
 If ``True``, the logs will just contain the root path. If it is set to ``False``
 then it displays the component responsible for the log output
 
+.. setting:: LOGSTATS_INTERVAL
+
+LOGSTATS_INTERVAL
+-----------------
+
+Default: ``60.0``
+
+The interval (in seconds) between each logging printout of the stats 
+by :class:`~extensions.logstats.LogStats`.
+
 .. setting:: MEMDEBUG_ENABLED
 
 MEMDEBUG_ENABLED
@@ -1155,9 +1165,14 @@ Type of in-memory queue used by scheduler. Other available type is:
 
 SCHEDULER_PRIORITY_QUEUE
 ------------------------
-Default: ``'queuelib.PriorityQueue'``
+Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
 
-Type of priority queue used by scheduler.
+Type of priority queue used by the scheduler. Another available type is
+``scrapy.pqueues.DownloaderAwarePriorityQueue``.
+``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
+``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
+domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue``
+does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
 
 .. setting:: SPIDER_CONTRACTS
 

docs/topics/spider-middleware.rst

@@ -54,6 +54,8 @@ value.  For example, if you want to disable the off-site middleware::
 Finally, keep in mind that some middlewares may need to be enabled through a
 particular setting. See each middleware documentation for more info.
 
+.. _custom-spider-middleware:
+
 Writing your own spider middleware
 ==================================
 

docs/topics/spiders.rst

@@ -661,7 +661,7 @@ SitemapSpider
 
     .. attribute:: sitemap_follow
 
-        A list of regexes of sitemap that should be followed. This is is only
+        A list of regexes of sitemap that should be followed. This is only
         for sites that use `Sitemap index files`_ that point to other sitemap
         files.
 

docs/topics/stats.rst

@@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select
 which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
 default Stats Collector used is the :class:`MemoryStatsCollector`. 
 
-.. module:: scrapy.statscollectors
-   :synopsis: Stats Collectors
+.. currentmodule:: scrapy.statscollectors
 
 MemoryStatsCollector
 --------------------

docs/topics/telnetconsole.rst

@@ -1,12 +1,11 @@
+.. currentmodule:: scrapy.extensions.telnet
+
 .. _topics-telnetconsole:
 
 ==============
 Telnet Console
 ==============
 
-.. module:: scrapy.extensions.telnet
-   :synopsis: The Telnet Console
-
 Scrapy comes with a built-in telnet console for inspecting and controlling a
 Scrapy running process. The telnet console is just a regular python shell
 running inside the Scrapy process, so you can do literally anything from it.
@@ -45,7 +44,7 @@ the console you need to type::
     >>>
 
 By default Username is ``scrapy`` and Password is autogenerated. The 
-autogenerated Password can be seen on scrapy logs like the example bellow::
+autogenerated Password can be seen on scrapy logs like the example below::
 
     2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
 

scrapy/cmdline.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
-import sys, os
+import sys
+import os
 import optparse
 import cProfile
 import inspect
@@ -14,6 +15,7 @@ from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
 from scrapy.settings.deprecated import check_deprecated_settings
 
+
 def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
     # scrapy.utils.spider.iter_spider_classes
@@ -25,6 +27,7 @@ def _iter_command_classes(module_name):
                     not obj == ScrapyCommand:
                 yield obj
 
+
 def _get_commands_from_module(module, inproject):
     d = {}
     for cmd in _iter_command_classes(module):
@@ -33,6 +36,7 @@ def _get_commands_from_module(module, inproject):
             d[cmdname] = cmd()
     return d
 
+
 def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
     cmds = {}
     for entry_point in pkg_resources.iter_entry_points(group):
@@ -43,6 +47,7 @@ def _get_commands_from_entry_points(inproject, group='scrapy.commands'):
             raise Exception("Invalid entry point %s" % entry_point.name)
     return cmds
 
+
 def _get_commands_dict(settings, inproject):
     cmds = _get_commands_from_module('scrapy.commands', inproject)
     cmds.update(_get_commands_from_entry_points(inproject))
@@ -51,6 +56,7 @@ def _get_commands_dict(settings, inproject):
         cmds.update(_get_commands_from_module(cmds_module, inproject))
     return cmds
 
+
 def _pop_command_name(argv):
     i = 0
     for arg in argv[1:]:
@@ -59,13 +65,15 @@ def _pop_command_name(argv):
             return arg
         i += 1
 
+
 def _print_header(settings, inproject):
     if inproject:
         print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
-            settings['BOT_NAME']))
+                                             settings['BOT_NAME']))
     else:
         print("Scrapy %s - no active project\n" % scrapy.__version__)
 
+
 def _print_commands(settings, inproject):
     _print_header(settings, inproject)
     print("Usage:")
@@ -80,11 +88,13 @@ def _print_commands(settings, inproject):
     print()
     print('Use "scrapy <command> -h" to see more info about a command')
 
+
 def _print_unknown_command(settings, cmdname, inproject):
     _print_header(settings, inproject)
     print("Unknown command: %s\n" % cmdname)
     print('Use "scrapy" to see available commands')
 
+
 def _run_print_help(parser, func, *a, **kw):
     try:
         func(*a, **kw)
@@ -95,41 +105,27 @@ def _run_print_help(parser, func, *a, **kw):
             parser.print_help()
         sys.exit(2)
 
+
 def execute(argv=None, settings=None):
     if argv is None:
         argv = sys.argv
 
-    # --- backward compatibility for scrapy.conf.settings singleton ---
-    if settings is None and 'scrapy.conf' in sys.modules:
-        from scrapy import conf
-        if hasattr(conf, 'settings'):
-            settings = conf.settings
-    # ------------------------------------------------------------------
-
     if settings is None:
         settings = get_project_settings()
         # set EDITOR from environment if available
         try:
             editor = os.environ['EDITOR']
-        except KeyError: pass
+        except KeyError:
+            pass
         else:
             settings['EDITOR'] = editor
     check_deprecated_settings(settings)
 
-    # --- backward compatibility for scrapy.conf.settings singleton ---
-    import warnings
-    from scrapy.exceptions import ScrapyDeprecationWarning
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", ScrapyDeprecationWarning)
-        from scrapy import conf
-        conf.settings = settings
-    # ------------------------------------------------------------------
-
     inproject = inside_project()
     cmds = _get_commands_dict(settings, inproject)
     cmdname = _pop_command_name(argv)
     parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
-        conflict_handler='resolve')
+                                   conflict_handler='resolve')
     if not cmdname:
         _print_commands(settings, inproject)
         sys.exit(0)
@@ -150,12 +146,14 @@ def execute(argv=None, settings=None):
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
 
+
 def _run_command(cmd, args, opts):
     if opts.profile:
         _run_command_profiled(cmd, args, opts)
     else:
         cmd.run(args, opts)
 
+
 def _run_command_profiled(cmd, args, opts):
     if opts.profile:
         sys.stderr.write("scrapy: writing cProfile stats to %r\n" % opts.profile)
@@ -165,6 +163,7 @@ def _run_command_profiled(cmd, args, opts):
     if opts.profile:
         p.dump_stats(opts.profile)
 
+
 if __name__ == '__main__':
     try:
         execute()

scrapy/commands/check.py

@@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResult
 
 from scrapy.commands import ScrapyCommand
 from scrapy.contracts import ContractsManager
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, set_environ
 from scrapy.utils.conf import build_component_list
 
 
@@ -68,16 +68,17 @@ class Command(ScrapyCommand):
 
         spider_loader = self.crawler_process.spider_loader
 
-        for spidername in args or spider_loader.list():
-            spidercls = spider_loader.load(spidername)
-            spidercls.start_requests = lambda s: conman.from_spider(s, result)
+        with set_environ(SCRAPY_CHECK='true'):
+            for spidername in args or spider_loader.list():
+                spidercls = spider_loader.load(spidername)
+                spidercls.start_requests = lambda s: conman.from_spider(s, result)
 
-            tested_methods = conman.tested_methods_from_spidercls(spidercls)
-            if opts.list:
-                for method in tested_methods:
-                    contract_reqs[spidercls.name].append(method)
-            elif tested_methods:
-                self.crawler_process.crawl(spidercls)
+                tested_methods = conman.tested_methods_from_spidercls(spidercls)
+                if opts.list:
+                    for method in tested_methods:
+                        contract_reqs[spidercls.name].append(method)
+                elif tested_methods:
+                    self.crawler_process.crawl(spidercls)
 
         # start checks
         if opts.list:

scrapy/commands/parse.py

@@ -51,12 +51,13 @@ class Command(ScrapyCommand):
             help="use this callback for parsing, instead looking for a callback")
         parser.add_option("-m", "--meta", dest="meta",
             help="inject extra meta into the Request, it must be a valid raw json string")
+        parser.add_option("--cbkwargs", dest="cbkwargs",
+            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_option("-d", "--depth", dest="depth", type="int", default=1,
             help="maximum depth for parsing requests [default: %default]")
         parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
             help="print each depth level one by one")
 
-
     @property
     def max_level(self):
         levels = list(self.items.keys()) + list(self.requests.keys())
@@ -111,10 +112,11 @@ class Command(ScrapyCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def run_callback(self, response, cb):
+    def run_callback(self, response, callback, cb_kwargs=None):
+        cb_kwargs = cb_kwargs or {}
         items, requests = [], []
 
-        for x in iterate_spider_output(cb(response)):
+        for x in iterate_spider_output(callback(response, **cb_kwargs)):
             if isinstance(x, (BaseItem, dict)):
                 items.append(x)
             elif isinstance(x, Request):
@@ -142,8 +144,7 @@ class Command(ScrapyCommand):
         else:
             self.spidercls = spidercls_for_request(spider_loader, Request(url))
             if not self.spidercls:
-                logger.error('Unable to find spider for: %(url)s',
-                             {'url': url})
+                logger.error('Unable to find spider for: %(url)s', {'url': url})
 
         # Request requires callback argument as callable or None, not string
         request = Request(url, None)
@@ -160,7 +161,7 @@ class Command(ScrapyCommand):
                          {'url': url})
 
     def prepare_request(self, spider, request, opts):
-        def callback(response):
+        def callback(response, **cb_kwargs):
             # memorize first request
             if not self.first_response:
                 self.first_response = response
@@ -175,7 +176,7 @@ class Command(ScrapyCommand):
 
                     if not cb:
                         logger.error('Cannot find a rule that matches %(url)r in spider: %(spider)s',
-                                 {'url': response.url, 'spider': spider.name})
+                                     {'url': response.url, 'spider': spider.name})
                         return
                 else:
                     cb = 'parse'
@@ -192,7 +193,7 @@ class Command(ScrapyCommand):
             # parse items and requests
             depth = response.meta['_depth']
 
-            items, requests = self.run_callback(response, cb)
+            items, requests = self.run_callback(response, cb, cb_kwargs)
             if opts.pipelines:
                 itemproc = self.pcrawler.engine.scraper.itemproc
                 for item in items:
@@ -207,10 +208,14 @@ class Command(ScrapyCommand):
                     req.callback = callback
                 return requests
 
-        #update request meta if any extra meta was passed through the --meta/-m opts.
+        # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
             request.meta.update(opts.meta)
 
+        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
+        if opts.cbkwargs:
+            request.cb_kwargs.update(opts.cbkwargs)
+
         request.meta['_depth'] = 1
         request.meta['_callback'] = request.callback
         request.callback = callback
@@ -221,23 +226,29 @@ class Command(ScrapyCommand):
 
         self.process_spider_arguments(opts)
         self.process_request_meta(opts)
+        self.process_request_cb_kwargs(opts)
 
     def process_spider_arguments(self, opts):
-
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
             raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
 
     def process_request_meta(self, opts):
-
         if opts.meta:
             try:
                 opts.meta = json.loads(opts.meta)
             except ValueError:
-                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. " \
-                                "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
+                raise UsageError("Invalid -m/--meta value, pass a valid json string to -m or --meta. "
+                                 "Example: --meta='{\"foo\" : \"bar\"}'", print_help=False)
 
+    def process_request_cb_kwargs(self, opts):
+        if opts.cbkwargs:
+            try:
+                opts.cbkwargs = json.loads(opts.cbkwargs)
+            except ValueError:
+                raise UsageError("Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
+                                 "Example: --cbkwargs='{\"foo\" : \"bar\"}'", print_help=False)
 
     def run(self, args, opts):
         # parse arguments

scrapy/conf.py

@@ -1,13 +0,0 @@
-# This module is kept for backward compatibility, so users can import
-# scrapy.conf.settings and get the settings they expect
-
-import sys
-
-if 'scrapy.cmdline' not in sys.modules:
-    from scrapy.utils.project import get_project_settings
-    settings = get_project_settings()
-
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.conf` is deprecated, use `crawler.settings` attribute instead",
-    ScrapyDeprecationWarning, stacklevel=2)

scrapy/contracts/__init__.py

@@ -94,7 +94,7 @@ class ContractsManager(object):
             try:
                 output = cb(response)
                 output = list(iterate_spider_output(output))
-            except:
+            except Exception:
                 case = _create_testcase(method, 'callback')
                 results.addError(case, sys.exc_info())
 

scrapy/core/downloader/__init__.py

@@ -60,10 +60,6 @@ class Slot(object):
 
 def _get_concurrency_delay(concurrency, spider, settings):
     delay = settings.getfloat('DOWNLOAD_DELAY')
-    if hasattr(spider, 'DOWNLOAD_DELAY'):
-        warnings.warn("%s.DOWNLOAD_DELAY attribute is deprecated, use %s.download_delay instead" %
-                      (type(spider).__name__, type(spider).__name__))
-        delay = spider.DOWNLOAD_DELAY
     if hasattr(spider, 'download_delay'):
         delay = spider.download_delay
 
@@ -75,6 +71,8 @@ def _get_concurrency_delay(concurrency, spider, settings):
 
 class Downloader(object):
 
+    DOWNLOAD_SLOT = 'download_slot'
+
     def __init__(self, crawler):
         self.settings = crawler.settings
         self.signals = crawler.signals
@@ -111,8 +109,8 @@ class Downloader(object):
         return key, self.slots[key]
 
     def _get_slot_key(self, request, spider):
-        if 'download_slot' in request.meta:
-            return request.meta['download_slot']
+        if self.DOWNLOAD_SLOT in request.meta:
+            return request.meta[self.DOWNLOAD_SLOT]
 
         key = urlparse_cached(request).hostname or ''
         if self.ip_concurrency:
@@ -122,7 +120,7 @@ class Downloader(object):
 
     def _enqueue_request(self, request, spider):
         key, slot = self._get_slot(request, spider)
-        request.meta['download_slot'] = key
+        request.meta[self.DOWNLOAD_SLOT] = key
 
         def _deactivate(response):
             slot.active.remove(request)

scrapy/core/downloader/handlers/http.py

@@ -1,15 +1,3 @@
 from __future__ import absolute_import
 from .http10 import HTTP10DownloadHandler
 from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
-
-
-# backward compatibility
-class HttpDownloadHandler(HTTP10DownloadHandler):
-
-    def __init__(self, *args, **kwargs):
-        import warnings
-        from scrapy.exceptions import ScrapyDeprecationWarning
-        warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader'
-                      '.handlers.http10.HTTP10DownloadHandler instead',
-                      category=ScrapyDeprecationWarning, stacklevel=1)
-        super(HttpDownloadHandler, self).__init__(*args, **kwargs)

scrapy/core/scheduler.py

@@ -1,19 +1,46 @@
 import os
 import json
 import logging
+import warnings
 from os.path import join, exists
 
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from queuelib import PriorityQueue
+
 from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.job import job_dir
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+
 
 logger = logging.getLogger(__name__)
 
 
 class Scheduler(object):
+    """
+    Scrapy Scheduler. It allows to enqueue requests and then get
+    a next request to download. Scheduler is also handling duplication
+    filtering, via dupefilter.
 
+    Prioritization and queueing is not performed by the Scheduler.
+    User sets ``priority`` field for each Request, and a PriorityQueue
+    (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities
+    to dequeue requests in a desired order.
+
+    Scheduler uses two PriorityQueue instances, configured to work in-memory
+    and on-disk (optional). When on-disk queue is present, it is used by
+    default, and an in-memory queue is used as a fallback for cases where
+    a disk queue can't handle a request (can't serialize it).
+
+    :setting:`SCHEDULER_MEMORY_QUEUE` and
+    :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes
+    which PriorityQueue instances would be instantiated with, to keep requests
+    on disk and in memory respectively.
+
+    Overall, Scheduler is an object which holds several PriorityQueue instances
+    (in-memory and on-disk) and implements fallback logic for them.
+    Also, it handles dupefilters.
+    """
     def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None,
-                 logunser=False, stats=None, pqclass=None):
+                 logunser=False, stats=None, pqclass=None, crawler=None):
         self.df = dupefilter
         self.dqdir = self._dqdir(jobdir)
         self.pqclass = pqclass
@@ -21,6 +48,7 @@ class Scheduler(object):
         self.mqclass = mqclass
         self.logunser = logunser
         self.stats = stats
+        self.crawler = crawler
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -28,26 +56,35 @@ class Scheduler(object):
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
         dupefilter = create_instance(dupefilter_cls, settings, crawler)
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
+        if pqclass is PriorityQueue:
+            warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
+                          " is no longer supported because of API changes; "
+                          "please use 'scrapy.pqueues.ScrapyPriorityQueue'",
+                          ScrapyDeprecationWarning)
+            from scrapy.pqueues import ScrapyPriorityQueue
+            pqclass = ScrapyPriorityQueue
+
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
-        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG'))
+        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS',
+                                    settings.getbool('SCHEDULER_DEBUG'))
         return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
-                   stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
+                   stats=crawler.stats, pqclass=pqclass, dqclass=dqclass,
+                   mqclass=mqclass, crawler=crawler)
 
     def has_pending_requests(self):
         return len(self) > 0
 
     def open(self, spider):
         self.spider = spider
-        self.mqs = self.pqclass(self._newmq)
+        self.mqs = self._mq()
         self.dqs = self._dq() if self.dqdir else None
         return self.df.open()
 
     def close(self, reason):
         if self.dqs:
-            prios = self.dqs.close()
-            with open(join(self.dqdir, 'active.json'), 'w') as f:
-                json.dump(prios, f)
+            state = self.dqs.close()
+            self._write_dqs_state(self.dqdir, state)
         return self.df.close(reason)
 
     def enqueue_request(self, request):
@@ -82,8 +119,7 @@ class Scheduler(object):
         if self.dqs is None:
             return
         try:
-            reqd = request_to_dict(request, self.spider)
-            self.dqs.push(reqd, -request.priority)
+            self.dqs.push(request, -request.priority)
         except ValueError as e:  # non serializable request
             if self.logunser:
                 msg = ("Unable to serialize request: %(request)s - reason:"
@@ -103,32 +139,51 @@ class Scheduler(object):
 
     def _dqpop(self):
         if self.dqs:
-            d = self.dqs.pop()
-            if d:
-                return request_from_dict(d, self.spider)
+            return self.dqs.pop()
 
     def _newmq(self, priority):
+        """ Factory for creating memory queues. """
         return self.mqclass()
 
     def _newdq(self, priority):
-        return self.dqclass(join(self.dqdir, 'p%s' % priority))
+        """ Factory for creating disk queues. """
+        path = join(self.dqdir, 'p%s' % (priority, ))
+        return self.dqclass(path)
+
+    def _mq(self):
+        """ Create a new priority queue instance, with in-memory storage """
+        return create_instance(self.pqclass, None, self.crawler, self._newmq,
+                               serialize=False)
 
     def _dq(self):
-        activef = join(self.dqdir, 'active.json')
-        if exists(activef):
-            with open(activef) as f:
-                prios = json.load(f)
-        else:
-            prios = ()
-        q = self.pqclass(self._newdq, startprios=prios)
+        """ Create a new priority queue instance, with disk storage """
+        state = self._read_dqs_state(self.dqdir)
+        q = create_instance(self.pqclass,
+                            None,
+                            self.crawler,
+                            self._newdq,
+                            state,
+                            serialize=True)
         if q:
             logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
                         {'queuesize': len(q)}, extra={'spider': self.spider})
         return q
 
     def _dqdir(self, jobdir):
+        """ Return a folder name to keep disk queue state at """
         if jobdir:
             dqdir = join(jobdir, 'requests.queue')
             if not exists(dqdir):
                 os.makedirs(dqdir)
             return dqdir
+
+    def _read_dqs_state(self, dqdir):
+        path = join(dqdir, 'active.json')
+        if not exists(path):
+            return ()
+        with open(path) as f:
+            return json.load(f)
+
+    def _write_dqs_state(self, dqdir, state):
+        with open(join(dqdir, 'active.json'), 'w') as f:
+            json.dump(state, f)

scrapy/core/scraper.py

@@ -142,7 +142,9 @@ class Scraper(object):
     def call_spider(self, result, request, spider):
         result.request = request
         dfd = defer_result(result)
-        dfd.addCallbacks(request.callback or spider.parse, request.errback)
+        dfd.addCallbacks(callback=request.callback or spider.parse,
+                         errback=request.errback,
+                         callbackKeywords=request.cb_kwargs)
         return dfd.addCallback(iterate_spider_output)
 
     def handle_spider_error(self, _failure, request, response, spider):

scrapy/core/spidermw.py

@@ -49,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                                              .format(fname(method), type(result)))
                 except _InvalidOutput:
                     raise
-                except:
+                except Exception:
                     return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 

scrapy/crawler.py

@@ -111,6 +111,8 @@ class Crawler(object):
 
     @defer.inlineCallbacks
     def stop(self):
+        """Starts a graceful stop of the crawler and returns a deferred that is
+        fired when the crawler is stopped."""
         if self.crawling:
             self.crawling = False
             yield defer.maybeDeferred(self.engine.stop)
@@ -330,14 +332,7 @@ class CrawlerProcess(CrawlerRunner):
 
 def _get_spider_loader(settings):
     """ Get SpiderLoader instance from settings """
-    if settings.get('SPIDER_MANAGER_CLASS'):
-        warnings.warn(
-            'SPIDER_MANAGER_CLASS option is deprecated. '
-            'Please use SPIDER_LOADER_CLASS.',
-            category=ScrapyDeprecationWarning, stacklevel=2
-        )
-    cls_path = settings.get('SPIDER_MANAGER_CLASS',
-                            settings.get('SPIDER_LOADER_CLASS'))
+    cls_path = settings.get('SPIDER_LOADER_CLASS')
     loader_cls = load_object(cls_path)
     try:
         verifyClass(ISpiderLoader, loader_cls)

scrapy/downloadermiddlewares/redirect.py

@@ -88,6 +88,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
 
     def __init__(self, settings):
         super(MetaRefreshMiddleware, self).__init__(settings)
+        self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
         self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY',
                                          settings.getint('METAREFRESH_MAXDELAY'))
 
@@ -96,7 +97,8 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
                 not isinstance(response, HtmlResponse):
             return response
 
-        interval, url = get_meta_refresh(response)
+        interval, url = get_meta_refresh(response,
+                                         ignore_tags=self._ignore_tags)
         if url and interval < self._maxdelay:
             redirected = self._redirect_request_using_get(request, url)
             return self._redirect(redirected, request, spider, 'meta refresh')

scrapy/extensions/corestats.py

@@ -24,7 +24,11 @@ class CoreStats(object):
         self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
 
     def spider_closed(self, spider, reason):
-        self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
+        finish_time = datetime.datetime.utcnow()
+        elapsed_time = finish_time - self.stats.get_value('start_time')
+        elapsed_time_seconds = elapsed_time.total_seconds()
+        self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
+        self.stats.set_value('finish_time', finish_time, spider=spider)
         self.stats.set_value('finish_reason', reason, spider=spider)
 
     def item_scraped(self, item, spider):

scrapy/extensions/feedexport.py

@@ -98,7 +98,8 @@ class S3FeedStorage(BlockingFeedStorage):
         # without using from_crawler)
         no_defaults = access_key is None and secret_key is None
         if no_defaults:
-            from scrapy.conf import settings
+            from scrapy.utils.project import get_project_settings
+            settings = get_project_settings()
             if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
                 import warnings
                 from scrapy.exceptions import ScrapyDeprecationWarning

scrapy/extensions/httpcache.py

@@ -31,7 +31,7 @@ class DummyPolicy(object):
     def should_cache_response(self, response, request):
         return response.status not in self.ignore_http_codes
 
-    def is_cached_response_fresh(self, response, request):
+    def is_cached_response_fresh(self, cachedresponse, request):
         return True
 
     def is_cached_response_valid(self, cachedresponse, response, request):
@@ -70,7 +70,7 @@ class RFC2616Policy(object):
         return True
 
     def should_cache_response(self, response, request):
-        # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1
+        # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
         # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
         # Status code 206 is not included because cache can not deal with partial contents
         cc = self._parse_cachecontrol(response)

scrapy/http/request/__init__.py

@@ -18,7 +18,7 @@ class Request(object_ref):
 
     def __init__(self, url, callback=None, method='GET', headers=None, body=None,
                  cookies=None, meta=None, encoding='utf-8', priority=0,
-                 dont_filter=False, errback=None, flags=None):
+                 dont_filter=False, errback=None, flags=None, cb_kwargs=None):
 
         self._encoding = encoding  # this one has to be set first
         self.method = str(method).upper()
@@ -40,8 +40,15 @@ class Request(object_ref):
         self.dont_filter = dont_filter
 
         self._meta = dict(meta) if meta else None
+        self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
         self.flags = [] if flags is None else list(flags)
 
+    @property
+    def cb_kwargs(self):
+        if self._cb_kwargs is None:
+            self._cb_kwargs = {}
+        return self._cb_kwargs
+
     @property
     def meta(self):
         if self._meta is None:
@@ -92,7 +99,7 @@ class Request(object_ref):
         given new values.
         """
         for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags',
-                  'encoding', 'priority', 'dont_filter', 'callback', 'errback']:
+                  'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop('cls', self.__class__)
         return cls(*args, **kwargs)

scrapy/http/request/form.py

@@ -18,6 +18,7 @@ from scrapy.utils.response import get_base_url
 
 
 class FormRequest(Request):
+    valid_form_methods = ['GET', 'POST']
 
     def __init__(self, *args, **kwargs):
         formdata = kwargs.pop('formdata', None)
@@ -48,7 +49,13 @@ class FormRequest(Request):
         form = _get_form(response, formname, formid, formnumber, formxpath)
         formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
         url = _get_form_url(form, kwargs.pop('url', None))
+
         method = kwargs.pop('method', form.method)
+        if method is not None:
+            method = method.upper()
+            if method not in cls.valid_form_methods:
+                method = 'GET'
+
         return cls(url=url, method=method, formdata=formdata, **kwargs)
 
 

scrapy/http/response/__init__.py

@@ -106,7 +106,7 @@ class Response(object_ref):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding='utf-8', priority=0,
-               dont_filter=False, errback=None):
+               dont_filter=False, errback=None, cb_kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -132,4 +132,5 @@ class Response(object_ref):
                        encoding=encoding,
                        priority=priority,
                        dont_filter=dont_filter,
-                       errback=errback)
+                       errback=errback,
+                       cb_kwargs=cb_kwargs)

scrapy/http/response/text.py

@@ -123,7 +123,7 @@ class TextResponse(Response):
 
     def follow(self, url, callback=None, method='GET', headers=None, body=None,
                cookies=None, meta=None, encoding=None, priority=0,
-               dont_filter=False, errback=None):
+               dont_filter=False, errback=None, cb_kwargs=None):
         # type: (...) -> Request
         """
         Return a :class:`~.Request` instance to follow a link ``url``.
@@ -154,7 +154,8 @@ class TextResponse(Response):
             encoding=encoding,
             priority=priority,
             dont_filter=dont_filter,
-            errback=errback
+            errback=errback,
+            cb_kwargs=cb_kwargs,
         )
 
 

scrapy/loader/__init__.py

@@ -8,8 +8,6 @@ import six
 
 from scrapy.item import Item
 from scrapy.selector import Selector
-from scrapy.utils.decorators import deprecated
-from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.misc import arg_to_iter, extract_regex
 from scrapy.utils.python import flatten
 
@@ -35,6 +33,10 @@ class ItemLoader(object):
         self.parent = parent
         self._local_item = context['item'] = item
         self._local_values = defaultdict(list)
+        # Preprocess values if item built from dict
+        # Values need to be added to item._values if added them from dict (not with add_values)
+        for field_name, value in item.items():
+            self._values[field_name] = self._process_input_value(field_name, value)
 
     @property
     def _values(self):
@@ -105,8 +107,14 @@ class ItemLoader(object):
         for proc in processors:
             if value is None:
                 break
+            _proc = proc
             proc = wrap_loader_context(proc, self.context)
-            value = proc(value)
+            try:
+                value = proc(value)
+            except Exception as e:
+                raise ValueError("Error with processor %s value=%r error='%s: %s'" %
+                                 (_proc.__class__.__name__, value,
+                                  type(e).__name__, str(e)))
         return value
 
     def load_item(self):
@@ -146,8 +154,15 @@ class ItemLoader(object):
 
     def _process_input_value(self, field_name, value):
         proc = self.get_input_processor(field_name)
+        _proc = proc
         proc = wrap_loader_context(proc, self.context)
-        return proc(value)
+        try:
+            return proc(value)
+        except Exception as e:
+            raise ValueError(
+                "Error with input processor %s: field=%r value=%r "
+                "error='%s: %s'" % (_proc.__class__.__name__, field_name,
+                                    value, type(e).__name__, str(e)))
 
     def _get_item_field_attr(self, field_name, key, default=None):
         if isinstance(self.item, Item):
@@ -174,10 +189,6 @@ class ItemLoader(object):
         values = self._get_xpathvalues(xpath, **kw)
         return self.get_value(values, *processors, **kw)
 
-    @deprecated(use_instead='._get_xpathvalues()')
-    def _get_values(self, xpaths, **kw):
-        return self._get_xpathvalues(xpaths, **kw)
-
     def _get_xpathvalues(self, xpaths, **kw):
         self._check_selector_method()
         xpaths = arg_to_iter(xpaths)
@@ -199,5 +210,3 @@ class ItemLoader(object):
         self._check_selector_method()
         csss = arg_to_iter(csss)
         return flatten(self.selector.css(css).getall() for css in csss)
-
-XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)

scrapy/loader/processors.py

@@ -25,7 +25,13 @@ class MapCompose(object):
         for func in wrapped_funcs:
             next_values = []
             for v in values:
-                next_values += arg_to_iter(func(v))
+                try:
+                    next_values += arg_to_iter(func(v))
+                except Exception as e:
+                    raise ValueError("Error in MapCompose with "
+                                     "%s value=%r error='%s: %s'" %
+                                     (str(func), value, type(e).__name__,
+                                      str(e)))
             values = next_values
         return values
 
@@ -46,7 +52,12 @@ class Compose(object):
         for func in wrapped_funcs:
             if value is None and self.stop_on_none:
                 break
-            value = func(value)
+            try:
+                value = func(value)
+            except Exception as e:
+                raise ValueError("Error in Compose with "
+                                 "%s value=%r error='%s: %s'" %
+                                 (str(func), value, type(e).__name__, str(e)))
         return value
 
 

scrapy/log.py

@@ -1,60 +0,0 @@
-"""
-This module is kept to provide a helpful warning about its removal.
-"""
-import logging
-import warnings
-
-from twisted.python.failure import Failure
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.log import failure_to_exc_info
-
-logger = logging.getLogger(__name__)
-
-warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
-              "the builtin Python library for logging. Read the updated "
-              "logging entry in the documentation to learn more.",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-
-# Imports and level_names variable kept for backward-compatibility
-
-DEBUG = logging.DEBUG
-INFO = logging.INFO
-WARNING = logging.WARNING
-ERROR = logging.ERROR
-CRITICAL = logging.CRITICAL
-SILENT = CRITICAL + 1
-
-level_names = {
-    logging.DEBUG: "DEBUG",
-    logging.INFO: "INFO",
-    logging.WARNING: "WARNING",
-    logging.ERROR: "ERROR",
-    logging.CRITICAL: "CRITICAL",
-    SILENT: "SILENT",
-}
-
-
-def msg(message=None, _level=logging.INFO, **kw):
-    warnings.warn('log.msg has been deprecated, create a python logger and '
-                  'log through it instead',
-                  ScrapyDeprecationWarning, stacklevel=2)
-
-    level = kw.pop('level', _level)
-    message = kw.pop('format', message)
-    # NOTE: logger.log doesn't handle well passing empty dictionaries with format
-    # arguments because of some weird use-case:
-    # https://hg.python.org/cpython/file/648dcafa7e5f/Lib/logging/__init__.py#l269
-    logger.log(level, message, *[kw] if kw else [])
-
-
-def err(_stuff=None, _why=None, **kw):
-    warnings.warn('log.err has been deprecated, create a python logger and '
-                  'use its error method instead',
-                  ScrapyDeprecationWarning, stacklevel=2)
-
-    level = kw.pop('level', logging.ERROR)
-    failure = kw.pop('failure', _stuff) or Failure()
-    message = kw.pop('why', _why) or failure.value
-    logger.log(level, message, *[kw] if kw else [], exc_info=failure_to_exc_info(failure))

scrapy/pipelines/files.py

@@ -458,33 +458,6 @@ class FilesPipeline(MediaPipeline):
         return item
 
     def file_path(self, request, response=None, info=None):
-        ## start of deprecation warning block (can be removed in the future)
-        def _warn():
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            import warnings
-            warnings.warn('FilesPipeline.file_key(url) method is deprecated, please use '
-                          'file_path(request, response=None, info=None) instead',
-                          category=ScrapyDeprecationWarning, stacklevel=1)
-
-        # check if called from file_key with url as first argument
-        if not isinstance(request, Request):
-            _warn()
-            url = request
-        else:
-            url = request.url
-
-        # detect if file_key() method has been overridden
-        if not hasattr(self.file_key, '_base'):
-            _warn()
-            return self.file_key(url)
-        ## end of deprecation warning block
-
-        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
-        media_ext = os.path.splitext(url)[1]  # change to request.url after deprecation
+        media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        media_ext = os.path.splitext(request.url)[1]
         return 'full/%s%s' % (media_guid, media_ext)
-
-    # deprecated
-    def file_key(self, url):
-        return self.file_path(url)
-
-    file_key._base = True

scrapy/pipelines/images.py

@@ -165,69 +165,9 @@ class ImagesPipeline(FilesPipeline):
         return item
 
     def file_path(self, request, response=None, info=None):
-        ## start of deprecation warning block (can be removed in the future)
-        def _warn():
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            import warnings
-            warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, '
-                          'please use file_path(request, response=None, info=None) instead',
-                          category=ScrapyDeprecationWarning, stacklevel=1)
-
-        # check if called from image_key or file_key with url as first argument
-        if not isinstance(request, Request):
-            _warn()
-            url = request
-        else:
-            url = request.url
-
-        # detect if file_key() or image_key() methods have been overridden
-        if not hasattr(self.file_key, '_base'):
-            _warn()
-            return self.file_key(url)
-        elif not hasattr(self.image_key, '_base'):
-            _warn()
-            return self.image_key(url)
-        ## end of deprecation warning block
-
-        image_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
+        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return 'full/%s.jpg' % (image_guid)
 
     def thumb_path(self, request, thumb_id, response=None, info=None):
-        ## start of deprecation warning block (can be removed in the future)
-        def _warn():
-            from scrapy.exceptions import ScrapyDeprecationWarning
-            import warnings
-            warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use '
-                          'thumb_path(request, thumb_id, response=None, info=None) instead',
-                          category=ScrapyDeprecationWarning, stacklevel=1)
-
-        # check if called from thumb_key with url as first argument
-        if not isinstance(request, Request):
-            _warn()
-            url = request
-        else:
-            url = request.url
-
-        # detect if thumb_key() method has been overridden
-        if not hasattr(self.thumb_key, '_base'):
-            _warn()
-            return self.thumb_key(url, thumb_id)
-        ## end of deprecation warning block
-
-        thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
+        thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid)
-
-    # deprecated
-    def file_key(self, url):
-        return self.image_key(url)
-    file_key._base = True
-
-    # deprecated
-    def image_key(self, url):
-        return self.file_path(url)
-    image_key._base = True
-
-    # deprecated
-    def thumb_key(self, url, thumb_id):
-        return self.thumb_path(url, thumb_id)
-    thumb_key._base = True

scrapy/pipelines/media.py

@@ -3,7 +3,7 @@ from __future__ import print_function
 import functools
 import logging
 from collections import defaultdict
-from twisted.internet.defer import Deferred, DeferredList
+from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
 from twisted.python.failure import Failure
 
 from scrapy.settings import Settings
@@ -139,6 +139,30 @@ class MediaPipeline(object):
             result.cleanFailure()
             result.frames = []
             result.stack = None
+
+            # This code fixes a memory leak by avoiding to keep references to
+            # the Request and Response objects on the Media Pipeline cache.
+            #
+            # Twisted inline callbacks pass return values using the function
+            # twisted.internet.defer.returnValue, which encapsulates the return
+            # value inside a _DefGen_Return base exception.
+            #
+            # What happens when the media_downloaded callback raises another
+            # exception, for example a FileException('download-error') when
+            # the Response status code is not 200 OK, is that it stores the
+            # _DefGen_Return exception on the FileException context.
+            #
+            # To avoid keeping references to the Response and therefore Request
+            # objects on the Media Pipeline cache, we should wipe the context of
+            # the exception encapsulated by the Twisted Failure when its a
+            # _DefGen_Return instance.
+            #
+            # This problem does not occur in Python 2.7 since we don't have
+            # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
+            context = getattr(result.value, '__context__', None)
+            if isinstance(context, _DefGen_Return):
+                setattr(result.value, '__context__', None)
+
         info.downloading.remove(fp)
         info.downloaded[fp] = result  # cache result
         for wad in info.waiting.pop(fp):

scrapy/pqueues.py

@@ -0,0 +1,193 @@
+import hashlib
+import logging
+from collections import namedtuple
+
+from queuelib import PriorityQueue
+
+from scrapy.utils.reqser import request_to_dict, request_from_dict
+
+
+logger = logging.getLogger(__name__)
+
+
+def _path_safe(text):
+    """
+    Return a filesystem-safe version of a string ``text``
+
+    >>> _path_safe('simple.org').startswith('simple.org')
+    True
+    >>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org')
+    True
+    >>> _path_safe('some@symbol?').startswith('some_symbol_')
+    True
+    """
+    pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_'
+                             for c in text])
+    # as we replace some letters we can get collision for different slots
+    # add we add unique part
+    unique_slot = hashlib.md5(text.encode('utf8')).hexdigest()
+    return '-'.join([pathable_slot, unique_slot])
+
+
+class _Priority(namedtuple("_Priority", ["priority", "slot"])):
+    """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple
+    which can be used instead of int priorities in queues:
+
+    * they are ordered in the same way - order is still by priority value,
+      min(prios) works;
+    * str(p) representation is guaranteed to be different when slots
+      are different - this is important because str(p) is used to create
+      queue files on disk;
+    * they have readable str(p) representation which is safe
+      to use as a file name.
+    """
+    __slots__ = ()
+
+    def __str__(self):
+        return '%s_%s' % (self.priority, _path_safe(str(self.slot)))
+
+
+class _SlotPriorityQueues(object):
+    """ Container for multiple priority queues. """
+    def __init__(self, pqfactory, slot_startprios=None):
+        """
+        ``pqfactory`` is a factory for creating new PriorityQueues.
+        It must be a function which accepts a single optional ``startprios``
+        argument, with a list of priorities to create queues for.
+
+        ``slot_startprios`` is a ``{slot: startprios}`` dict.
+        """
+        self.pqfactory = pqfactory
+        self.pqueues = {}  # slot -> priority queue
+        for slot, startprios in (slot_startprios or {}).items():
+            self.pqueues[slot] = self.pqfactory(startprios)
+
+    def pop_slot(self, slot):
+        """ Pop an object from a priority queue for this slot """
+        queue = self.pqueues[slot]
+        request = queue.pop()
+        if len(queue) == 0:
+            del self.pqueues[slot]
+        return request
+
+    def push_slot(self, slot, obj, priority):
+        """ Push an object to a priority queue for this slot """
+        if slot not in self.pqueues:
+            self.pqueues[slot] = self.pqfactory()
+        queue = self.pqueues[slot]
+        queue.push(obj, priority)
+
+    def close(self):
+        active = {slot: queue.close()
+                  for slot, queue in self.pqueues.items()}
+        self.pqueues.clear()
+        return active
+
+    def __len__(self):
+        return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
+
+    def __contains__(self, slot):
+        return slot in self.pqueues
+
+
+class ScrapyPriorityQueue(PriorityQueue):
+    """
+    PriorityQueue which works with scrapy.Request instances and
+    can optionally convert them to/from dicts before/after putting to a queue.
+    """
+    def __init__(self, crawler, qfactory, startprios=(), serialize=False):
+        super(ScrapyPriorityQueue, self).__init__(qfactory, startprios)
+        self.serialize = serialize
+        self.spider = crawler.spider
+
+    @classmethod
+    def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False):
+        return cls(crawler, qfactory, startprios, serialize)
+
+    def push(self, request, priority=0):
+        if self.serialize:
+            request = request_to_dict(request, self.spider)
+        super(ScrapyPriorityQueue, self).push(request, priority)
+
+    def pop(self):
+        request = super(ScrapyPriorityQueue, self).pop()
+        if request and self.serialize:
+            request = request_from_dict(request, self.spider)
+        return request
+
+
+class DownloaderInterface(object):
+
+    def __init__(self, crawler):
+        self.downloader = crawler.engine.downloader
+
+    def stats(self, possible_slots):
+        return [(self._active_downloads(slot), slot)
+                for slot in possible_slots]
+
+    def get_slot_key(self, request):
+        return self.downloader._get_slot_key(request, None)
+
+    def _active_downloads(self, slot):
+        """ Return a number of requests in a Downloader for a given slot """
+        if slot not in self.downloader.slots:
+            return 0
+        return len(self.downloader.slots[slot].active)
+
+
+class DownloaderAwarePriorityQueue(object):
+    """ PriorityQueue which takes Downlaoder activity in account:
+    domains (slots) with the least amount of active downloads are dequeued
+    first.
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False):
+        return cls(crawler, qfactory, slot_startprios, serialize)
+
+    def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False):
+        if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
+            raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP'
+                             % (self.__class__,))
+
+        if slot_startprios and not isinstance(slot_startprios, dict):
+            raise ValueError("DownloaderAwarePriorityQueue accepts "
+                             "``slot_startprios`` as a dict; %r instance "
+                             "is passed. Most likely, it means the state is"
+                             "created by an incompatible priority queue. "
+                             "Only a crawl started with the same priority "
+                             "queue class can be resumed." %
+                             slot_startprios.__class__)
+
+        slot_startprios = {
+            slot: [_Priority(p, slot) for p in startprios]
+            for slot, startprios in (slot_startprios or {}).items()}
+
+        def pqfactory(startprios=()):
+            return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize)
+        self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios)
+        self.serialize = serialize
+        self._downloader_interface = DownloaderInterface(crawler)
+
+    def pop(self):
+        stats = self._downloader_interface.stats(self._slot_pqueues.pqueues)
+
+        if not stats:
+            return
+
+        slot = min(stats)[1]
+        request = self._slot_pqueues.pop_slot(slot)
+        return request
+
+    def push(self, request, priority):
+        slot = self._downloader_interface.get_slot_key(request)
+        priority_slot = _Priority(priority=priority, slot=slot)
+        self._slot_pqueues.push_slot(slot, request, priority_slot)
+
+    def close(self):
+        active = self._slot_pqueues.close()
+        return {slot: [p.priority for p in startprios]
+                for slot, startprios in active.items()}
+
+    def __len__(self):
+        return len(self._slot_pqueues)

scrapy/selector/__init__.py

@@ -2,4 +2,3 @@
 Selectors
 """
 from scrapy.selector.unified import *
-from scrapy.selector.lxmlsel import *

scrapy/selector/csstranslator.py

@@ -1,15 +0,0 @@
-from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
-from scrapy.utils.deprecate import create_deprecated_class
-
-
-ScrapyXPathExpr = create_deprecated_class(
-    'ScrapyXPathExpr', XPathExpr,
-    new_class_path='parsel.csstranslator.XPathExpr')
-
-ScrapyGenericTranslator = create_deprecated_class(
-    'ScrapyGenericTranslator', GenericTranslator,
-    new_class_path='parsel.csstranslator.GenericTranslator')
-
-ScrapyHTMLTranslator = create_deprecated_class(
-    'ScrapyHTMLTranslator', HTMLTranslator,
-    new_class_path='parsel.csstranslator.HTMLTranslator')

scrapy/selector/lxmlsel.py

@@ -1,50 +0,0 @@
-"""
-XPath selectors based on lxml
-"""
-from scrapy.utils.deprecate import create_deprecated_class
-from .unified import Selector, SelectorList
-
-
-__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector',
-           'XPathSelectorList']
-
-def _xpathselector_css(self, *a, **kw):
-    raise RuntimeError('.css() method not available for %s, '
-                        'instantiate scrapy.Selector '
-                        'instead' % type(self).__name__)
-
-XPathSelector = create_deprecated_class(
-    'XPathSelector',
-    Selector,
-    {
-        '__slots__': (),
-        '_default_type': 'html',
-        'css': _xpathselector_css,
-    },
-    new_class_path='scrapy.Selector',
-    old_class_path='scrapy.selector.XPathSelector',
-)
-
-XmlXPathSelector = create_deprecated_class(
-    'XmlXPathSelector',
-    XPathSelector,
-    clsdict={
-        '__slots__': (),
-        '_default_type': 'xml',
-    },
-    new_class_path='scrapy.Selector',
-    old_class_path='scrapy.selector.XmlXPathSelector',
-)
-
-HtmlXPathSelector = create_deprecated_class(
-    'HtmlXPathSelector',
-    XPathSelector,
-    clsdict={
-        '__slots__': (),
-        '_default_type': 'html',
-    },
-    new_class_path='scrapy.Selector',
-    old_class_path='scrapy.selector.HtmlXPathSelector',
-)
-
-XPathSelectorList = create_deprecated_class('XPathSelectorList', SelectorList)

scrapy/selector/unified.py

@@ -8,7 +8,6 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.python import to_bytes
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.utils.decorators import deprecated
-from scrapy.exceptions import ScrapyDeprecationWarning
 
 
 __all__ = ['Selector', 'SelectorList']
@@ -31,17 +30,6 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
     The :class:`SelectorList` class is a subclass of the builtin ``list``
     class, which provides a few additional methods.
     """
-    @deprecated(use_instead='.extract()')
-    def extract_unquoted(self):
-        return [x.extract_unquoted() for x in self]
-
-    @deprecated(use_instead='.xpath()')
-    def x(self, xpath):
-        return self.select(xpath)
-
-    @deprecated(use_instead='.xpath()')
-    def select(self, xpath):
-        return self.xpath(xpath)
 
 
 class Selector(_ParselSelector, object_ref):
@@ -78,21 +66,13 @@ class Selector(_ParselSelector, object_ref):
     __slots__ = ['response']
     selectorlist_cls = SelectorList
 
-    def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
+    def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
         if not(response is None or text is None):
            raise ValueError('%s.__init__() received both response and text'
                             % self.__class__.__name__)
 
         st = _st(response, type or self._default_type)
 
-        if _root is not None:
-            warnings.warn("Argument `_root` is deprecated, use `root` instead",
-                          ScrapyDeprecationWarning, stacklevel=2)
-            if root is None:
-                root = _root
-            else:
-                warnings.warn("Ignoring deprecated `_root` argument, using provided `root`")
-
         if text is not None:
             response = _response_from_text(text, st)
 
@@ -102,18 +82,3 @@ class Selector(_ParselSelector, object_ref):
 
         self.response = response
         super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
-
-    # Deprecated api
-    @property
-    def _root(self):
-        warnings.warn("Attribute `_root` is deprecated, use `root` instead",
-                      ScrapyDeprecationWarning, stacklevel=2)
-        return self.root
-
-    @deprecated(use_instead='.xpath()')
-    def select(self, xpath):
-        return self.xpath(xpath)
-
-    @deprecated(use_instead='.extract()')
-    def extract_unquoted(self):
-        return self.extract()

scrapy/settings/default_settings.py

@@ -221,6 +221,7 @@ MEMUSAGE_NOTIFY_MAIL = []
 MEMUSAGE_WARNING_MB = 0
 
 METAREFRESH_ENABLED = True
+METAREFRESH_IGNORE_TAGS = ['script', 'noscript']
 METAREFRESH_MAXDELAY = 100
 
 NEWSPIDER_MODULE = ''
@@ -238,7 +239,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408]
+RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False
@@ -246,7 +247,7 @@ ROBOTSTXT_OBEY = False
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
-SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
+SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue'
 
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 SPIDER_LOADER_WARN_ONLY = False

scrapy/spiders/__init__.py

@@ -10,7 +10,6 @@ from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
-from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.deprecate import method_is_overridden
 
@@ -52,15 +51,6 @@ class Spider(object_ref):
         spider._set_crawler(crawler)
         return spider
 
-    def set_crawler(self, crawler):
-        warnings.warn("set_crawler is deprecated, instantiate and bound the "
-                      "spider to this crawler with from_crawler method "
-                      "instead.",
-                      category=ScrapyDeprecationWarning, stacklevel=2)
-        assert not hasattr(self, 'crawler'), "Spider already bounded to a " \
-                                             "crawler"
-        self._set_crawler(crawler)
-
     def _set_crawler(self, crawler):
         self.crawler = crawler
         self.settings = crawler.settings
@@ -109,22 +99,6 @@ class Spider(object_ref):
     __repr__ = __str__
 
 
-BaseSpider = create_deprecated_class('BaseSpider', Spider)
-
-
-class ObsoleteClass(object):
-    def __init__(self, message):
-        self.message = message
-
-    def __getattr__(self, name):
-        raise AttributeError(self.message)
-
-spiders = ObsoleteClass(
-    '"from scrapy.spider import spiders" no longer works - use '
-    '"from scrapy.spiderloader import SpiderLoader" and instantiate '
-    'it with your project settings"'
-)
-
 # Top-level imports
 from scrapy.spiders.crawl import CrawlSpider, Rule
 from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider

scrapy/spiders/crawl.py

@@ -119,7 +119,3 @@ class CrawlSpider(Spider):
         spider._follow_links = crawler.settings.getbool(
             'CRAWLSPIDER_FOLLOW_LINKS', True)
         return spider
-
-    def set_crawler(self, crawler):
-        super(CrawlSpider, self).set_crawler(crawler)
-        self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)

scrapy/squeues.py

@@ -7,6 +7,7 @@ from six.moves import cPickle as pickle
 
 from queuelib import queue
 
+
 def _serializable_queue(queue_class, serialize, deserialize):
 
     class SerializableQueue(queue_class):
@@ -22,6 +23,7 @@ def _serializable_queue(queue_class, serialize, deserialize):
 
     return SerializableQueue
 
+
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
@@ -31,13 +33,14 @@ def _pickle_serialize(obj):
     except (pickle.PicklingError, AttributeError, TypeError) as e:
         raise ValueError(str(e))
 
-PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
+
+PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
     _pickle_serialize, pickle.loads)
-PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
+PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
     _pickle_serialize, pickle.loads)
-MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \
+MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
     marshal.dumps, marshal.loads)
-MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \
+MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
     marshal.dumps, marshal.loads)
 FifoMemoryQueue = queue.FifoMemoryQueue
 LifoMemoryQueue = queue.LifoMemoryQueue

scrapy/telnet.py

@@ -1,7 +0,0 @@
-import warnings
-from scrapy.exceptions import ScrapyDeprecationWarning
-warnings.warn("Module `scrapy.telnet` is deprecated, "
-              "use `scrapy.extensions.telnet` instead",
-              ScrapyDeprecationWarning, stacklevel=2)
-
-from scrapy.extensions.telnet import *

scrapy/templates/project/module/middlewares.py.tmpl

@@ -39,7 +39,7 @@ class ${ProjectName}SpiderMiddleware(object):
         # Called when a spider or process_spider_input() method
         # (from other spider middleware) raises an exception.
 
-        # Should return either None or an iterable of Response, dict
+        # Should return either None or an iterable of Request, dict
         # or Item objects.
         pass
 

scrapy/utils/defer.py

@@ -48,7 +48,7 @@ def mustbe_deferred(f, *args, **kw):
     # exception in Scrapy - see #125
     except IgnoreRequest as e:
         return defer_fail(failure.Failure(e))
-    except:
+    except Exception:
         return defer_fail(failure.Failure())
     else:
         return defer_result(result)
@@ -102,5 +102,5 @@ def iter_errback(iterable, errback, *a, **kw):
             yield next(it)
         except StopIteration:
             break
-        except:
+        except Exception:
             errback(failure.Failure(), *a, **kw)

scrapy/utils/gz.py

@@ -9,6 +9,9 @@ from gzip import GzipFile
 import six
 import re
 
+from scrapy.utils.decorators import deprecated
+
+
 # - Python>=3.5 GzipFile's read() has issues returning leftover
 #   uncompressed data when input is corrupted
 #   (regression or bug-fix compared to Python 3.4)
@@ -53,6 +56,7 @@ def gunzip(data):
 _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
 _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
 
+@deprecated
 def is_gzipped(response):
     """Return True if the response is gzipped, or False otherwise"""
     ctype = response.headers.get('Content-Type', b'')

scrapy/utils/http.py

@@ -4,8 +4,19 @@ Transitional module for moving to the w3lib library.
 For new code, always import from w3lib.http instead of this module
 """
 
+import warnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.decorators import deprecated
 from w3lib.http import *
 
+
+warnings.warn("Module `scrapy.utils.http` is deprecated, "
+              "Please import from `w3lib.http` instead.",
+              ScrapyDeprecationWarning, stacklevel=2)
+
+
+@deprecated
 def decode_chunked_transfer(chunked_body):
     """Parsed body received with chunked transfer encoding, and return the
     decoded body.

scrapy/utils/markup.py

@@ -3,5 +3,12 @@ Transitional module for moving to the w3lib library.
 
 For new code, always import from w3lib.html instead of this module
 """
+import warnings
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from w3lib.html import *
+
+
+warnings.warn("Module `scrapy.utils.markup` is deprecated. "
+              "Please import from `w3lib.html` instead.",
+              ScrapyDeprecationWarning, stacklevel=2)

scrapy/utils/misc.py

@@ -1,6 +1,8 @@
 """Helper functions which don't fit anywhere else"""
+import os
 import re
 import hashlib
+from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
 
@@ -86,7 +88,7 @@ def extract_regex(regex, text, encoding='utf-8'):
 
     try:
         strings = [regex.search(text).group('extract')]   # named group
-    except:
+    except Exception:
         strings = regex.findall(text)    # full regex or numbered groups
     strings = flatten(strings)
 
@@ -142,3 +144,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
         return objcls.from_settings(settings, *args, **kwargs)
     else:
         return objcls(*args, **kwargs)
+
+
+@contextmanager
+def set_environ(**kwargs):
+    """Temporarily set environment variables inside the context manager and
+    fully restore previous environment afterwards
+    """
+
+    original_env = {k: os.environ.get(k) for k in kwargs}
+    os.environ.update(kwargs)
+    try:
+        yield
+    finally:
+        for k, v in original_env.items():
+            if v is None:
+                del os.environ[k]
+            else:
+                os.environ[k] = v

scrapy/utils/multipart.py

@@ -3,5 +3,13 @@ Transitional module for moving to the w3lib library.
 
 For new code, always import from w3lib.form instead of this module
 """
+import warnings
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from w3lib.form import *
+
+
+warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
+              "If you're using `encode_multipart` function, please use "
+              "`urllib3.filepost.encode_multipart_formdata` instead",
+              ScrapyDeprecationWarning, stacklevel=2)

scrapy/utils/python.py

@@ -84,19 +84,6 @@ def unique(list_, key=lambda x: x):
     return result
 
 
-@deprecated("scrapy.utils.python.to_unicode")
-def str_to_unicode(text, encoding=None, errors='strict'):
-    """ This function is deprecated.
-    Please use scrapy.utils.python.to_unicode. """
-    return to_unicode(text, encoding, errors)
-
-
-@deprecated("scrapy.utils.python.to_bytes")
-def unicode_to_str(text, encoding=None, errors='strict'):
-    """ This function is deprecated. Please use scrapy.utils.python.to_bytes """
-    return to_bytes(text, encoding, errors)
-
-
 def to_unicode(text, encoding=None, errors='strict'):
     """Return the unicode representation of a bytes object ``text``. If
     ``text`` is already an unicode object, return it as-is."""

scrapy/utils/reqser.py

@@ -32,7 +32,8 @@ def request_to_dict(request, spider=None):
         '_encoding': request._encoding,
         'priority': request.priority,
         'dont_filter': request.dont_filter,
-        'flags': request.flags
+        'flags': request.flags,
+        'cb_kwargs': request.cb_kwargs,
     }
     if type(request) is not Request:
         d['_class'] = request.__module__ + '.' + request.__class__.__name__
@@ -64,7 +65,23 @@ def request_from_dict(d, spider=None):
         encoding=d['_encoding'],
         priority=d['priority'],
         dont_filter=d['dont_filter'],
-        flags=d.get('flags'))
+        flags=d.get('flags'),
+        cb_kwargs=d.get('cb_kwargs'),
+    )
+
+
+def _is_private_method(name):
+    return name.startswith('__') and not name.endswith('__')
+
+
+def _mangle_private_name(obj, func, name):
+    qualname = getattr(func, '__qualname__', None)
+    if qualname is None:
+        classname = obj.__class__.__name__.lstrip('_')
+        return '_%s%s' % (classname, name)
+    else:
+        splits = qualname.split('.')
+        return '_%s%s' % (splits[-2], splits[-1])
 
 
 def _find_method(obj, func):
@@ -75,7 +92,10 @@ def _find_method(obj, func):
             pass
         else:
             if func_self is obj:
-                return six.get_method_function(func).__name__
+                name = six.get_method_function(func).__name__
+                if _is_private_method(name):
+                    return _mangle_private_name(obj, func, name)
+                return name
     raise ValueError("Function %s is not a method of: %s" % (func, obj))
 
 

scrapy/utils/response.py

@@ -11,14 +11,6 @@ from twisted.web import http
 from scrapy.utils.python import to_bytes, to_native_str
 from w3lib import html
 
-from scrapy.utils.decorators import deprecated
-
-
-@deprecated
-def body_or_str(*a, **kw):
-    from scrapy.utils.iterators import _body_or_str
-    return _body_or_str(*a, **kw)
-
 
 _baseurl_cache = weakref.WeakKeyDictionary()
 def get_base_url(response):
@@ -31,12 +23,12 @@ def get_base_url(response):
 
 
 _metaref_cache = weakref.WeakKeyDictionary()
-def get_meta_refresh(response):
+def get_meta_refresh(response, ignore_tags=('script', 'noscript')):
     """Parse the http-equiv refrsh parameter from the given response"""
     if response not in _metaref_cache:
         text = response.text[0:4096]
         _metaref_cache[response] = html.get_meta_refresh(text, response.url,
-            response.encoding, ignore_tags=('script', 'noscript'))
+            response.encoding, ignore_tags=ignore_tags)
     return _metaref_cache[response]
 
 

setup.py

@@ -65,7 +65,8 @@ setup(
     ],
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
     install_requires=[
-        'Twisted>=13.1.0',
+        'Twisted>=13.1.0;python_version!="3.4"',
+        'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',
         'w3lib>=1.17.0',
         'queuelib',
         'lxml',

tests/mockserver.py

@@ -177,7 +177,7 @@ class Root(Resource):
         try:
             from tests import tests_datadir
             self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
-        except:
+        except Exception:
             pass
         self.putChild(b"redirect-to", RedirectTo())
 

tests/requirements-py2.txt

@@ -2,9 +2,10 @@
 mock
 mitmproxy==0.10.1
 netlib==0.10.1
-pytest==2.9.2
+pytest
+pytest-cov
 pytest-twisted
-pytest-cov==2.2.1
+pytest-xdist
 jmespath
 brotlipy
 testfixtures

tests/requirements-py3.txt

@@ -1,6 +1,7 @@
-pytest==3.6.3
+pytest
+pytest-cov
 pytest-twisted
-pytest-cov==2.5.1
+pytest-xdist
 testfixtures
 jmespath
 leveldb; sys_platform != "win32"

tests/test_closespider.py

@@ -53,9 +53,5 @@ class TestCloseSpider(TestCase):
         yield crawler.crawl(total=1000000, mockserver=self.mockserver)
         reason = crawler.spider.meta['close_reason']
         self.assertEqual(reason, 'closespider_timeout')
-        stats = crawler.stats
-        start = stats.get_value('start_time')
-        stop = stats.get_value('finish_time')
-        diff = stop - start
-        total_seconds = diff.seconds + diff.microseconds
+        total_seconds = crawler.stats.get_value('elapsed_time_seconds')
         self.assertTrue(total_seconds >= close_on)

tests/test_command_parse.py

@@ -43,6 +43,12 @@ class MySpider(scrapy.Spider):
         else:
             self.logger.debug('It Works!')
 
+    def parse_request_with_cb_kwargs(self, response, foo=None, key=None):
+        if foo == 'bar' and key == 'value':
+            self.logger.debug('It Works!')
+        else:
+            self.logger.debug('It Does Not Work :(')
+
     def parse_request_without_meta(self, response):
         foo = response.meta.get('foo', 'bar')
 
@@ -120,6 +126,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_request_with_cb_kwargs(self):
+        raw_json_string = '{"foo" : "bar", "key": "value"}'
+        _, _, stderr = yield self.execute(['--spider', self.spider_name,
+                                           '--cbkwargs', raw_json_string,
+                                           '-c', 'parse_request_with_cb_kwargs',
+                                           self.url('/html')])
+        self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
     @defer.inlineCallbacks
     def test_request_without_meta(self):

tests/test_crawler.py

@@ -1,5 +1,4 @@
 import logging
-import tempfile
 import warnings
 
 from twisted.internet import defer
@@ -38,7 +37,11 @@ class CrawlerTestCase(BaseCrawlerTest):
             self.assertIsInstance(spiders, sl_cls)
 
             self.crawler.spiders
-            self.assertEqual(len(w), 1, "Warn deprecated access only once")
+            is_one_warning = len(w) == 1
+            if not is_one_warning:
+                for warning in w:
+                    print(warning)
+            self.assertTrue(is_one_warning, "Warn deprecated access only once")
 
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
@@ -173,23 +176,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
             sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
             self.assertIsInstance(spiders, sl_cls)
 
-    def test_spidermanager_deprecation(self):
-        with warnings.catch_warnings(record=True) as w:
-            runner = CrawlerRunner({
-                'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader'
-            })
-            self.assertIsInstance(runner.spider_loader, CustomSpiderLoader)
-            self.assertEqual(len(w), 1)
-            self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
-
-    def test_crawl_rejects_spider_objects(self):
-        with raises(ValueError):
-            CrawlerRunner().crawl(DefaultSpider())
-
-    def test_create_crawler_rejects_spider_objects(self):
-        with raises(ValueError):
-            CrawlerRunner().create_crawler(DefaultSpider())
-
 
 class CrawlerProcessTest(BaseCrawlerTest):
     def test_crawler_process_accepts_dict(self):

tests/test_downloader_handlers.py

@@ -24,7 +24,7 @@ from w3lib.url import path_to_file_uri
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
-from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
@@ -360,11 +360,6 @@ class HttpTestCase(unittest.TestCase):
         return d
 
 
-class DeprecatedHttpTestCase(HttpTestCase):
-    """HTTP 1.0 test case"""
-    download_handler_cls = HttpDownloadHandler
-
-
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
     download_handler_cls = HTTP10DownloadHandler
@@ -656,11 +651,6 @@ class HttpProxyTestCase(unittest.TestCase):
         return self.download_request(request, Spider('foo')).addCallback(_test)
 
 
-class DeprecatedHttpProxyTestCase(unittest.TestCase):
-    """Old deprecated reference to http10 downloader handler"""
-    download_handler_cls = HttpDownloadHandler
-
-
 class Http10ProxyTestCase(HttpProxyTestCase):
     download_handler_cls = HTTP10DownloadHandler
 

tests/test_downloadermiddleware.py

@@ -123,7 +123,6 @@ class ProcessRequestInvalidOutput(ManagerTestCase):
 
     def test_invalid_process_request(self):
         req = Request('http://example.com/index.html')
-        resp = Response('http://example.com/index.html')
 
         class InvalidProcessRequestMiddleware:
             def process_request(self, request, spider):
@@ -143,7 +142,6 @@ class ProcessResponseInvalidOutput(ManagerTestCase):
 
     def test_invalid_process_response(self):
         req = Request('http://example.com/index.html')
-        resp = Response('http://example.com/index.html')
 
         class InvalidProcessResponseMiddleware:
             def process_response(self, request, response, spider):
@@ -163,7 +161,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase):
 
     def test_invalid_process_exception(self):
         req = Request('http://example.com/index.html')
-        resp = Response('http://example.com/index.html')
 
         class InvalidProcessExceptionMiddleware:
             def process_request(self, request, spider):

tests/test_downloadermiddleware_redirect.py

@@ -279,5 +279,24 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
         self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh'])
         self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh'])
 
+    def test_ignore_tags_default(self):
+        req = Request(url='http://example.org')
+        body = ('''<noscript><meta http-equiv="refresh" '''
+                '''content="0;URL='http://example.org/newpage'"></noscript>''')
+        rsp = HtmlResponse(req.url, body=body.encode())
+        response = self.mw.process_response(req, rsp, self.spider)
+        assert isinstance(response, Response)
+
+    def test_ignore_tags_empty_list(self):
+        crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []})
+        mw = MetaRefreshMiddleware.from_crawler(crawler)
+        req = Request(url='http://example.org')
+        body = ('''<noscript><meta http-equiv="refresh" '''
+                '''content="0;URL='http://example.org/newpage'"></noscript>''')
+        rsp = HtmlResponse(req.url, body=body.encode())
+        req2 = mw.process_response(req, rsp, self.spider)
+        assert isinstance(req2, Request)
+        self.assertEqual(req2.url, 'http://example.org/newpage')
+
 if __name__ == "__main__":
     unittest.main()

tests/test_feedexport.py

@@ -26,6 +26,7 @@ from scrapy.extensions.feedexport import (
     BlockingFeedStorage)
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
 from scrapy.utils.python import to_native_str
+from scrapy.utils.project import get_project_settings
 
 
 class FileFeedStorageTest(unittest.TestCase):
@@ -134,8 +135,10 @@ class BlockingFeedStorageTest(unittest.TestCase):
 
 class S3FeedStorageTest(unittest.TestCase):
 
-    @mock.patch('scrapy.conf.settings', new={'AWS_ACCESS_KEY_ID': 'conf_key',
-                'AWS_SECRET_ACCESS_KEY': 'conf_secret'}, create=True)
+    @mock.patch('scrapy.utils.project.get_project_settings',
+                new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key',
+                                                 'AWS_SECRET_ACCESS_KEY': 'conf_secret'}),
+                create=True)
     def test_parse_credentials(self):
         try:
             import boto

tests/test_http_request.py

@@ -181,6 +181,7 @@ class RequestTest(unittest.TestCase):
         r1 = self.request_class("http://www.example.com", flags=['f1', 'f2'],
                                 callback=somecallback, errback=somecallback)
         r1.meta['foo'] = 'bar'
+        r1.cb_kwargs['key'] = 'value'
         r2 = r1.copy()
 
         # make sure copy does not propagate callbacks
@@ -193,6 +194,10 @@ class RequestTest(unittest.TestCase):
         assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
         self.assertEqual(r1.flags, r2.flags)
 
+        # make sure cb_kwargs dict is shallow copied
+        assert r1.cb_kwargs is not r2.cb_kwargs, "cb_kwargs must be a shallow copy, not identical"
+        self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
+
         # make sure meta dict is shallow copied
         assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
         self.assertEqual(r1.meta, r2.meta)
@@ -1100,6 +1105,20 @@ class FormRequestTest(RequestTest):
         self.assertRaises(ValueError, self.request_class.from_response,
                           response, formcss="input[name='abc']")
 
+    def test_from_response_valid_form_methods(self):
+        body = """<form action="post.php" method="%s">
+            <input type="hidden" name="one" value="1">
+            </form>"""
+
+        for method in self.request_class.valid_form_methods:
+            response = _buildresponse(body % method)
+            r = self.request_class.from_response(response)
+            self.assertEqual(r.method, method)
+
+        response = _buildresponse(body % 'UNKNOWN')
+        r = self.request_class.from_response(response)
+        self.assertEqual(r.method, 'GET')
+
 
 def _buildresponse(body, **kwargs):
     kwargs.setdefault('body', body)

tests/test_loader.py

@@ -419,6 +419,79 @@ class BasicItemLoaderTest(unittest.TestCase):
         self.assertEqual(item['url'], u'rabbit.hole')
         self.assertEqual(item['summary'], u'rabbithole')
 
+    def test_create_item_from_dict(self):
+        class TestItem(Item):
+            title = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        input_item = {'title': 'Test item title 1'}
+        il = TestItemLoader(item=input_item)
+        # Getting output value mustn't remove value from item
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+        self.assertEqual(il.get_output_value('title'), 'Test item title 1')
+        self.assertEqual(il.load_item(), {
+            'title': 'Test item title 1',
+        })
+
+        input_item = {'title': 'Test item title 2'}
+        il = TestItemLoader(item=input_item)
+        # Values from dict must be added to item _values
+        self.assertEqual(il._values.get('title'), 'Test item title 2')
+
+        input_item = {'title': [u'Test item title 3', u'Test item 4']}
+        il = TestItemLoader(item=input_item)
+        # Same rules must work for lists
+        self.assertEqual(il._values.get('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+        self.assertEqual(il.get_output_value('title'),
+                         [u'Test item title 3', u'Test item 4'])
+        self.assertEqual(il.load_item(), {
+            'title': [u'Test item title 3', u'Test item 4'],
+        })
+
+    def test_error_input_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_in = MapCompose(float)
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'])
+
+    def test_error_output_processor(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+            name_out = Compose(Join(), float)
+
+        il = TestItemLoader()
+        il.add_value('name', u'marta')
+        with self.assertRaises(ValueError):
+            il.load_item()
+
+    def test_error_processor_as_argument(self):
+        class TestItem(Item):
+            name = Field()
+
+        class TestItemLoader(ItemLoader):
+            default_item_class = TestItem
+
+        il = TestItemLoader()
+        self.assertRaises(ValueError, il.add_value, 'name',
+                          [u'marta', u'other'], Compose(float))
+
 
 class ProcessorsTest(unittest.TestCase):
 
@@ -445,13 +518,22 @@ class ProcessorsTest(unittest.TestCase):
         proc = Compose(str.upper)
         self.assertEqual(proc(None), None)
         proc = Compose(str.upper, stop_on_none=False)
-        self.assertRaises(TypeError, proc, None)
+        self.assertRaises(ValueError, proc, None)
+        proc = Compose(str.upper, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
 
     def test_mapcompose(self):
         filter_world = lambda x: None if x == 'world' else x
         proc = MapCompose(filter_world, six.text_type.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
+        proc = MapCompose(filter_world, six.text_type.upper)
+        self.assertEqual(proc(None), [])
+        proc = MapCompose(filter_world, six.text_type.upper)
+        self.assertRaises(ValueError, proc, [1])
+        proc = MapCompose(filter_world, lambda x: x + 1)
+        self.assertRaises(ValueError, proc, 'hello')
+
 
 
 class SelectortemLoaderTest(unittest.TestCase):

tests/test_pipeline_files.py

@@ -108,44 +108,6 @@ class FilesPipelineTestCase(unittest.TestCase):
             p.stop()
 
 
-class DeprecatedFilesPipeline(FilesPipeline):
-    def file_key(self, url):
-        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()
-        media_ext = os.path.splitext(url)[1]
-        return 'empty/%s%s' % (media_guid, media_ext)
-
-
-class DeprecatedFilesPipelineTestCase(unittest.TestCase):
-    def setUp(self):
-        self.tempdir = mkdtemp()
-
-    def init_pipeline(self, pipeline_class):
-        self.pipeline = pipeline_class.from_settings(Settings({'FILES_STORE': self.tempdir}))
-        self.pipeline.download_func = _mocked_download_func
-        self.pipeline.open_spider(None)
-
-    def test_default_file_key_method(self):
-        self.init_pipeline(FilesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.pdf"),
-                             'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message))
-
-    def test_overridden_file_key_method(self):
-        self.init_pipeline(DeprecatedFilesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
-                             'empty/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message))
-
-    def tearDown(self):
-        rmtree(self.tempdir)
-
-
 class FilesPipelineTestCaseFields(unittest.TestCase):
 
     def test_item_fields_default(self):

tests/test_pipeline_images.py

@@ -118,63 +118,6 @@ class DeprecatedImagesPipeline(ImagesPipeline):
         return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
 
 
-class DeprecatedImagesPipelineTestCase(unittest.TestCase):
-    def setUp(self):
-        self.tempdir = mkdtemp()
-
-    def init_pipeline(self, pipeline_class):
-        self.pipeline = pipeline_class(self.tempdir, download_func=_mocked_download_func)
-        self.pipeline.open_spider(None)
-
-    def test_default_file_key_method(self):
-        self.init_pipeline(ImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.gif"),
-                             'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
-
-    def test_default_image_key_method(self):
-        self.init_pipeline(ImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.image_key("https://dev.mydeco.com/mydeco.gif"),
-                             'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
-
-    def test_overridden_file_key_method(self):
-        self.init_pipeline(DeprecatedImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.gif")),
-                             'empty/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
-
-    def test_default_thumb_key_method(self):
-        self.init_pipeline(ImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.thumb_key("file:///tmp/foo.jpg", 50),
-                             'thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message))
-
-    def test_overridden_thumb_key_method(self):
-        self.init_pipeline(DeprecatedImagesPipeline)
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            self.assertEqual(self.pipeline.thumb_path(Request("file:///tmp/foo.jpg"), 50),
-                             'thumbsup/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
-            self.assertEqual(len(w), 1)
-            self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message))
-
-    def tearDown(self):
-        rmtree(self.tempdir)
-
-
 class ImagesPipelineTestCaseFields(unittest.TestCase):
 
     def test_item_fields_default(self):

tests/test_pipeline_media.py

@@ -1,15 +1,19 @@
 from __future__ import print_function
+
+import sys
+
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
 from twisted.internet import reactor
-from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
 
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.request import request_fingerprint
 from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.files import FileException
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
 from scrapy import signals
@@ -90,6 +94,77 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         self.pipe._modify_media_request(request)
         assert request.meta == {'handle_httpstatus_all': True}
 
+    def test_should_remove_req_res_references_before_caching_the_results(self):
+        """Regression test case to prevent a memory leak in the Media Pipeline.
+
+        The memory leak is triggered when an exception is raised when a Response
+        scheduled by the Media Pipeline is being returned. For example, when a
+        FileException('download-error') is raised because the Response status
+        code is not 200 OK.
+
+        It happens because we are keeping a reference to the Response object
+        inside the FileException context. This is caused by the way Twisted
+        return values from inline callbacks. It raises a custom exception
+        encapsulating the original return value.
+
+        The solution is to remove the exception context when this context is a
+        _DefGen_Return instance, the BaseException used by Twisted to pass the
+        returned value from those inline callbacks.
+
+        Maybe there's a better and more reliable way to test the case described
+        here, but it would be more complicated and involve running - or at least
+        mocking - some async steps from the Media Pipeline. The current test
+        case is simple and detects the problem very fast. On the other hand, it
+        would not detect another kind of leak happening due to old object
+        references being kept inside the Media Pipeline cache.
+
+        This problem does not occur in Python 2.7 since we don't have Exception
+        Chaining (https://www.python.org/dev/peps/pep-3134/).
+        """
+        # Create sample pair of Request and Response objects
+        request = Request('http://url')
+        response = Response('http://url', body=b'', request=request)
+
+        # Simulate the Media Pipeline behavior to produce a Twisted Failure
+        try:
+            # Simulate a Twisted inline callback returning a Response
+            # The returnValue method raises an exception encapsulating the value
+            returnValue(response)
+        except BaseException as exc:
+            def_gen_return_exc = exc
+            try:
+                # Simulate the media_downloaded callback raising a FileException
+                # This usually happens when the status code is not 200 OK
+                raise FileException('download-error')
+            except Exception as exc:
+                file_exc = exc
+                # Simulate Twisted capturing the FileException
+                # It encapsulates the exception inside a Twisted Failure
+                failure = Failure(file_exc)
+
+        # The Failure should encapsulate a FileException ...
+        self.assertEqual(failure.value, file_exc)
+        # ... and if we're running on Python 3 ...
+        if sys.version_info.major >= 3:
+            # ... it should have the returnValue exception set as its context
+            self.assertEqual(failure.value.__context__, def_gen_return_exc)
+
+        # Let's calculate the request fingerprint and fake some runtime data...
+        fp = request_fingerprint(request)
+        info = self.pipe.spiderinfo
+        info.downloading.add(fp)
+        info.waiting[fp] = []
+
+        # When calling the method that caches the Request's result ...
+        self.pipe._cache_result_and_execute_waiters(failure, fp, info)
+        # ... it should store the Twisted Failure ...
+        self.assertEqual(info.downloaded[fp], failure)
+        # ... encapsulating the original FileException ...
+        self.assertEqual(info.downloaded[fp].value, file_exc)
+        # ... but it should not store the returnValue exception on its context
+        context = getattr(info.downloaded[fp].value, '__context__', None)
+        self.assertIsNone(context)
+
 
 class MockedMediaPipeline(MediaPipeline):
 

tests/test_request_cb_kwargs.py

@@ -0,0 +1,169 @@
+from testfixtures import LogCapture
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+import six
+
+from scrapy.http import Request
+from scrapy.crawler import CrawlerRunner
+from tests.spiders import MockServerSpider
+from tests.mockserver import MockServer
+
+
+class InjectArgumentsDownloaderMiddleware(object):
+    """
+    Make sure downloader middlewares are able to update the keyword arguments
+    """
+    def process_request(self, request, spider):
+        if request.callback.__name__ == 'parse_downloader_mw':
+            request.cb_kwargs['from_process_request'] = True
+        return None
+
+    def process_response(self, request, response, spider):
+        if request.callback.__name__ == 'parse_downloader_mw':
+            request.cb_kwargs['from_process_response'] = True
+        return response
+
+
+class InjectArgumentsSpiderMiddleware(object):
+    """
+    Make sure spider middlewares are able to update the keyword arguments
+    """
+    def process_start_requests(self, start_requests, spider):
+        for request in start_requests:
+            if request.callback.__name__ == 'parse_spider_mw':
+                request.cb_kwargs['from_process_start_requests'] = True
+            yield request
+
+    def process_spider_input(self, response, spider):
+        request = response.request
+        if request.callback.__name__ == 'parse_spider_mw':
+            request.cb_kwargs['from_process_spider_input'] = True
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        for element in result:
+            if isinstance(element, Request) and element.callback.__name__ == 'parse_spider_mw_2':
+                element.cb_kwargs['from_process_spider_output'] = True
+            yield element
+
+
+class KeywordArgumentsSpider(MockServerSpider):
+    name = 'kwargs'
+    custom_settings = {
+        'DOWNLOADER_MIDDLEWARES': {
+            __name__ + '.InjectArgumentsDownloaderMiddleware': 750,
+        },
+        'SPIDER_MIDDLEWARES': {
+            __name__ + '.InjectArgumentsSpiderMiddleware': 750,
+        },
+    }
+
+    checks = list()
+
+    def start_requests(self):
+        data = {'key': 'value', 'number': 123}
+        yield Request(self.mockserver.url('/first'), self.parse_first, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_with'), self.parse_general, cb_kwargs=data)
+        yield Request(self.mockserver.url('/general_without'), self.parse_general)
+        yield Request(self.mockserver.url('/no_kwargs'), self.parse_no_kwargs)
+        yield Request(self.mockserver.url('/default'), self.parse_default, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_less'), self.parse_takes_less, cb_kwargs=data)
+        yield Request(self.mockserver.url('/takes_more'), self.parse_takes_more, cb_kwargs=data)
+        yield Request(self.mockserver.url('/downloader_mw'), self.parse_downloader_mw)
+        yield Request(self.mockserver.url('/spider_mw'), self.parse_spider_mw)
+
+    def parse_first(self, response, key, number):
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.crawler.stats.inc_value('boolean_checks', 2)
+        yield response.follow(
+            self.mockserver.url('/two'),
+            self.parse_second,
+            cb_kwargs={'new_key': 'new_value'})
+
+    def parse_second(self, response, new_key):
+        self.checks.append(new_key == 'new_value')
+        self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_general(self, response, **kwargs):
+        if response.url.endswith('/general_with'):
+            self.checks.append(kwargs['key'] == 'value')
+            self.checks.append(kwargs['number'] == 123)
+            self.crawler.stats.inc_value('boolean_checks', 2)
+        elif response.url.endswith('/general_without'):
+            self.checks.append(kwargs == {})
+            self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_no_kwargs(self, response):
+        self.checks.append(response.url.endswith('/no_kwargs'))
+        self.crawler.stats.inc_value('boolean_checks')
+
+    def parse_default(self, response, key, number=None, default=99):
+        self.checks.append(response.url.endswith('/default'))
+        self.checks.append(key == 'value')
+        self.checks.append(number == 123)
+        self.checks.append(default == 99)
+        self.crawler.stats.inc_value('boolean_checks', 4)
+
+    def parse_takes_less(self, response, key):
+        """
+        Should raise
+        TypeError: parse_takes_less() got an unexpected keyword argument 'number'
+        """
+
+    def parse_takes_more(self, response, key, number, other):
+        """
+        Should raise
+        TypeError: parse_takes_more() missing 1 required positional argument: 'other'
+        """
+
+    def parse_downloader_mw(self, response, from_process_request, from_process_response):
+        self.checks.append(bool(from_process_request))
+        self.checks.append(bool(from_process_response))
+        self.crawler.stats.inc_value('boolean_checks', 2)
+
+    def parse_spider_mw(self, response, from_process_spider_input, from_process_start_requests):
+        self.checks.append(bool(from_process_spider_input))
+        self.checks.append(bool(from_process_start_requests))
+        self.crawler.stats.inc_value('boolean_checks', 2)
+        return Request(self.mockserver.url('/spider_mw_2'), self.parse_spider_mw_2)
+
+    def parse_spider_mw_2(self, response, from_process_spider_output):
+        self.checks.append(bool(from_process_spider_output))
+        self.crawler.stats.inc_value('boolean_checks', 1)
+
+
+class CallbackKeywordArgumentsTestCase(TestCase):
+
+    maxDiff = None
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.runner = CrawlerRunner()
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    @defer.inlineCallbacks
+    def test_callback_kwargs(self):
+        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+        self.assertTrue(all(crawler.spider.checks))
+        self.assertEqual(len(crawler.spider.checks), crawler.stats.get_value('boolean_checks'))
+        # check exceptions for argument mismatch
+        exceptions = {}
+        for line in log.records:
+            for key in ('takes_less', 'takes_more'):
+                if key in line.getMessage():
+                    exceptions[key] = line
+        self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
+        self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
+        self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
+        # py2 and py3 messages are different
+        exc_message = str(exceptions['takes_more'].exc_info[1])
+        if six.PY2:
+            self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
+        elif six.PY3:
+            self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")

tests/test_scheduler.py

@@ -0,0 +1,342 @@
+import shutil
+import tempfile
+import unittest
+import collections
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+from scrapy.crawler import Crawler
+from scrapy.core.downloader import Downloader
+from scrapy.core.scheduler import Scheduler
+from scrapy.http import Request
+from scrapy.spiders import Spider
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
+
+
+MockEngine = collections.namedtuple('MockEngine', ['downloader'])
+MockSlot = collections.namedtuple('MockSlot', ['active'])
+
+
+class MockDownloader(object):
+    def __init__(self):
+        self.slots = dict()
+
+    def _get_slot_key(self, request, spider):
+        if Downloader.DOWNLOAD_SLOT in request.meta:
+            return request.meta[Downloader.DOWNLOAD_SLOT]
+
+        return urlparse_cached(request).hostname or ''
+
+    def increment(self, slot_key):
+        slot = self.slots.setdefault(slot_key, MockSlot(active=list()))
+        slot.active.append(1)
+
+    def decrement(self, slot_key):
+        slot = self.slots.get(slot_key)
+        slot.active.pop()
+
+    def close(self):
+        pass
+
+
+class MockCrawler(Crawler):
+    def __init__(self, priority_queue_cls, jobdir):
+
+        settings = dict(
+                LOG_UNSERIALIZABLE_REQUESTS=False,
+                SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue',
+                SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue',
+                SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
+                JOBDIR=jobdir,
+                DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter'
+                )
+        super(MockCrawler, self).__init__(Spider, settings)
+        self.engine = MockEngine(downloader=MockDownloader())
+
+
+class SchedulerHandler(object):
+    priority_queue_cls = None
+    jobdir = None
+
+    def create_scheduler(self):
+        self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir)
+        self.scheduler = Scheduler.from_crawler(self.mock_crawler)
+        self.spider = Spider(name='spider')
+        self.scheduler.open(self.spider)
+
+    def close_scheduler(self):
+        self.scheduler.close('finished')
+        self.mock_crawler.stop()
+        self.mock_crawler.engine.downloader.close()
+
+    def setUp(self):
+        self.create_scheduler()
+
+    def tearDown(self):
+        self.close_scheduler()
+
+
+_PRIORITIES = [("http://foo.com/a", -2),
+               ("http://foo.com/d", 1),
+               ("http://foo.com/b", -1),
+               ("http://foo.com/c", 0),
+               ("http://foo.com/e", 2)]
+
+
+_URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"}
+
+
+class BaseSchedulerInMemoryTester(SchedulerHandler):
+    def test_length(self):
+        self.assertFalse(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), 0)
+
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        self.assertTrue(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), len(_URLS))
+
+    def test_dequeue(self):
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        urls = set()
+        while self.scheduler.has_pending_requests():
+            urls.add(self.scheduler.next_request().url)
+
+        self.assertEqual(urls, _URLS)
+
+    def test_dequeue_priorities(self):
+        for url, priority in _PRIORITIES:
+            self.scheduler.enqueue_request(Request(url, priority=priority))
+
+        priorities = list()
+        while self.scheduler.has_pending_requests():
+            priorities.append(self.scheduler.next_request().priority)
+
+        self.assertEqual(priorities,
+                         sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+
+
+class BaseSchedulerOnDiskTester(SchedulerHandler):
+
+    def setUp(self):
+        self.jobdir = tempfile.mkdtemp()
+        self.create_scheduler()
+
+    def tearDown(self):
+        self.close_scheduler()
+
+        shutil.rmtree(self.jobdir)
+        self.jobdir = None
+
+    def test_length(self):
+        self.assertFalse(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), 0)
+
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        self.assertTrue(self.scheduler.has_pending_requests())
+        self.assertEqual(len(self.scheduler), len(_URLS))
+
+    def test_dequeue(self):
+        for url in _URLS:
+            self.scheduler.enqueue_request(Request(url))
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        urls = set()
+        while self.scheduler.has_pending_requests():
+            urls.add(self.scheduler.next_request().url)
+
+        self.assertEqual(urls, _URLS)
+
+    def test_dequeue_priorities(self):
+        for url, priority in _PRIORITIES:
+            self.scheduler.enqueue_request(Request(url, priority=priority))
+
+        self.close_scheduler()
+        self.create_scheduler()
+
+        priorities = list()
+        while self.scheduler.has_pending_requests():
+            priorities.append(self.scheduler.next_request().priority)
+
+        self.assertEqual(priorities,
+                         sorted([x[1] for x in _PRIORITIES], key=lambda x: -x))
+
+
+class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase):
+    priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
+
+
+class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase):
+    priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
+
+
+_URLS_WITH_SLOTS = [("http://foo.com/a", 'a'),
+                    ("http://foo.com/b", 'a'),
+                    ("http://foo.com/c", 'b'),
+                    ("http://foo.com/d", 'b'),
+                    ("http://foo.com/e", 'c'),
+                    ("http://foo.com/f", 'c')]
+
+
+class TestMigration(unittest.TestCase):
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tmpdir)
+
+    def _migration(self, tmp_dir):
+        prev_scheduler_handler = SchedulerHandler()
+        prev_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue'
+        prev_scheduler_handler.jobdir = tmp_dir
+
+        prev_scheduler_handler.create_scheduler()
+        for url in _URLS:
+            prev_scheduler_handler.scheduler.enqueue_request(Request(url))
+        prev_scheduler_handler.close_scheduler()
+
+        next_scheduler_handler = SchedulerHandler()
+        next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+        next_scheduler_handler.jobdir = tmp_dir
+
+        next_scheduler_handler.create_scheduler()
+
+    def test_migration(self):
+        with self.assertRaises(ValueError):
+            self._migration(self.tmpdir)
+
+
+def _is_scheduling_fair(enqueued_slots, dequeued_slots):
+    """
+    We enqueued same number of requests for every slot.
+    Assert correct order, e.g.
+
+    >>> enqueued = ['a', 'b', 'c'] * 2
+    >>> correct = ['a', 'c', 'b', 'b', 'a', 'c']
+    >>> incorrect = ['a', 'a', 'b', 'c', 'c', 'b']
+    >>> _is_scheduling_fair(enqueued, correct)
+    True
+    >>> _is_scheduling_fair(enqueued, incorrect)
+    False
+    """
+    if len(dequeued_slots) != len(enqueued_slots):
+        return False
+
+    slots_number = len(set(enqueued_slots))
+    for i in range(0, len(dequeued_slots), slots_number):
+        part = dequeued_slots[i:i + slots_number]
+        if len(part) != len(set(part)):
+            return False
+
+    return True
+
+
+class DownloaderAwareSchedulerTestMixin(object):
+    priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
+    reopen = False
+
+    def test_logic(self):
+        for url, slot in _URLS_WITH_SLOTS:
+            request = Request(url)
+            request.meta[Downloader.DOWNLOAD_SLOT] = slot
+            self.scheduler.enqueue_request(request)
+
+        if self.reopen:
+            self.close_scheduler()
+            self.create_scheduler()
+
+        dequeued_slots = list()
+        requests = []
+        downloader = self.mock_crawler.engine.downloader
+        while self.scheduler.has_pending_requests():
+            request = self.scheduler.next_request()
+            # pylint: disable=protected-access
+            slot = downloader._get_slot_key(request, None)
+            dequeued_slots.append(slot)
+            downloader.increment(slot)
+            requests.append(request)
+
+        for request in requests:
+            # pylint: disable=protected-access
+            slot = downloader._get_slot_key(request, None)
+            downloader.decrement(slot)
+
+        self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS),
+                                            dequeued_slots))
+        self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0)
+
+
+class TestSchedulerWithDownloaderAwareInMemory(DownloaderAwareSchedulerTestMixin,
+                                               BaseSchedulerInMemoryTester,
+                                               unittest.TestCase):
+    pass
+
+
+class TestSchedulerWithDownloaderAwareOnDisk(DownloaderAwareSchedulerTestMixin,
+                                             BaseSchedulerOnDiskTester,
+                                             unittest.TestCase):
+    reopen = True
+
+
+class StartUrlsSpider(Spider):
+
+    def __init__(self, start_urls):
+        self.start_urls = start_urls
+        super(StartUrlsSpider, self).__init__(start_urls)
+
+    def parse(self, response):
+        pass
+
+
+class TestIntegrationWithDownloaderAwareInMemory(TestCase):
+    def setUp(self):
+        self.crawler = get_crawler(
+                    StartUrlsSpider,
+                    {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
+                     'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
+                    )
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.crawler.stop()
+
+    @defer.inlineCallbacks
+    def test_integration_downloader_aware_priority_queue(self):
+        with MockServer() as mockserver:
+
+            url = mockserver.url("/status?n=200", is_secure=False)
+            start_urls = [url] * 6
+            yield self.crawler.crawl(start_urls)
+            self.assertEqual(self.crawler.stats.get_value('downloader/response_count'),
+                             len(start_urls))
+
+
+class TestIncompatibility(unittest.TestCase):
+
+    def _incompatible(self):
+        settings = dict(
+                SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
+                CONCURRENT_REQUESTS_PER_IP=1
+                )
+        crawler = Crawler(Spider, settings)
+        scheduler = Scheduler.from_crawler(crawler)
+        spider = Spider(name='spider')
+        scheduler.open(spider)
+
+    def test_incompatibility(self):
+        with self.assertRaises(ValueError):
+            self._incompatible()

tests/test_selector.py

@@ -3,7 +3,6 @@ import weakref
 from twisted.trial import unittest
 from scrapy.http import TextResponse, HtmlResponse, XmlResponse
 from scrapy.selector import Selector
-from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
 from lxml import etree
 
 
@@ -40,22 +39,6 @@ class SelectorTestCase(unittest.TestCase):
         sel = Selector(response)
         self.assertEqual(url, sel.root.base)
 
-    def test_deprecated_root_argument(self):
-        with warnings.catch_warnings(record=True) as w:
-            root = etree.fromstring(u'<html/>')
-            sel = Selector(_root=root)
-            self.assertIs(root, sel.root)
-            self.assertEqual(str(w[-1].message),
-                             'Argument `_root` is deprecated, use `root` instead')
-
-    def test_deprecated_root_argument_ambiguous(self):
-        with warnings.catch_warnings(record=True) as w:
-            _root = etree.fromstring(u'<xml/>')
-            root = etree.fromstring(u'<html/>')
-            sel = Selector(_root=_root, root=root)
-            self.assertIs(root, sel.root)
-            self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
-
     def test_flavor_detection(self):
         text = b'<div><img src="a.jpg"><p>Hello</div>'
         sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
@@ -101,111 +84,6 @@ class SelectorTestCase(unittest.TestCase):
         assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
             x.__class__.__name__
 
-    def test_deprecated_selector_methods(self):
-        sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.select('//p')
-            self.assertSubstring('Use .xpath() instead', str(w[-1].message))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.extract_unquoted()
-            self.assertSubstring('Use .extract() instead', str(w[-1].message))
-
-    def test_deprecated_selectorlist_methods(self):
-        sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.xpath('//p').select('.')
-            self.assertSubstring('Use .xpath() instead', str(w[-1].message))
-
-        with warnings.catch_warnings(record=True) as w:
-            sel.xpath('//p').extract_unquoted()
-            self.assertSubstring('Use .extract() instead', str(w[-1].message))
-
     def test_selector_bad_args(self):
         with self.assertRaisesRegexp(ValueError, 'received both response and text'):
             Selector(TextResponse(url='http://example.com', body=b''), text=u'')
-
-
-class DeprecatedXpathSelectorTest(unittest.TestCase):
-
-    text = '<div><img src="a.jpg"><p>Hello</div>'
-
-    def test_warnings_xpathselector(self):
-        cls = XPathSelector
-        with warnings.catch_warnings(record=True) as w:
-            class UserClass(cls):
-                pass
-
-            # subclassing must issue a warning
-            self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.Selector', str(w[0].message))
-
-            # subclass instance doesn't issue a warning
-            usel = UserClass(text=self.text)
-            self.assertEqual(len(w), 1)
-
-            # class instance must issue a warning
-            sel = cls(text=self.text)
-            self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.Selector', str(w[1].message))
-
-            # subclass and instance checks
-            self.assertTrue(issubclass(cls, Selector))
-            self.assertTrue(isinstance(sel, Selector))
-            self.assertTrue(isinstance(usel, Selector))
-
-    def test_warnings_xmlxpathselector(self):
-        cls = XmlXPathSelector
-        with warnings.catch_warnings(record=True) as w:
-            class UserClass(cls):
-                pass
-
-            # subclassing must issue a warning
-            self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.Selector', str(w[0].message))
-
-            # subclass instance doesn't issue a warning
-            usel = UserClass(text=self.text)
-            self.assertEqual(len(w), 1)
-
-            # class instance must issue a warning
-            sel = cls(text=self.text)
-            self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.Selector', str(w[1].message))
-
-            # subclass and instance checks
-            self.assertTrue(issubclass(cls, Selector))
-            self.assertTrue(issubclass(cls, XPathSelector))
-            self.assertTrue(isinstance(sel, Selector))
-            self.assertTrue(isinstance(usel, Selector))
-            self.assertTrue(isinstance(sel, XPathSelector))
-            self.assertTrue(isinstance(usel, XPathSelector))
-
-    def test_warnings_htmlxpathselector(self):
-        cls = HtmlXPathSelector
-        with warnings.catch_warnings(record=True) as w:
-            class UserClass(cls):
-                pass
-
-            # subclassing must issue a warning
-            self.assertEqual(len(w), 1, str(cls))
-            self.assertIn('scrapy.Selector', str(w[0].message))
-
-            # subclass instance doesn't issue a warning
-            usel = UserClass(text=self.text)
-            self.assertEqual(len(w), 1)
-
-            # class instance must issue a warning
-            sel = cls(text=self.text)
-            self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
-            self.assertIn('scrapy.Selector', str(w[1].message))
-
-            # subclass and instance checks
-            self.assertTrue(issubclass(cls, Selector))
-            self.assertTrue(issubclass(cls, XPathSelector))
-            self.assertTrue(isinstance(sel, Selector))
-            self.assertTrue(isinstance(usel, Selector))
-            self.assertTrue(isinstance(sel, XPathSelector))
-            self.assertTrue(isinstance(usel, XPathSelector))

tests/test_selector_csstranslator.py

@@ -1,22 +0,0 @@
-"""
-Selector tests for cssselect backend
-"""
-import warnings
-from twisted.trial import unittest
-from scrapy.selector.csstranslator import (
-    ScrapyHTMLTranslator,
-    ScrapyGenericTranslator,
-    ScrapyXPathExpr
-)
-
-
-class DeprecatedClassesTest(unittest.TestCase):
-
-    def test_deprecated_warnings(self):
-        for cls in [ScrapyHTMLTranslator, ScrapyGenericTranslator, ScrapyXPathExpr]:
-            with warnings.catch_warnings(record=True) as w:
-                obj = cls()
-                self.assertIn('%s is deprecated' % cls.__name__, str(w[-1].message),
-                              'Missing deprecate warning for %s' % cls.__name__)
-
-

tests/test_spider.py

@@ -10,7 +10,7 @@ from scrapy import signals
 from scrapy.settings import Settings
 from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
 from scrapy.spiders.init import InitSpider
-from scrapy.spiders import Spider, BaseSpider, CrawlSpider, Rule, XMLFeedSpider, \
+from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
     CSVFeedSpider, SitemapSpider
 from scrapy.linkextractors import LinkExtractor
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -51,17 +51,6 @@ class SpiderTest(unittest.TestCase):
         self.assertRaises(ValueError, self.spider_class)
         self.assertRaises(ValueError, self.spider_class, somearg='foo')
 
-    def test_deprecated_set_crawler_method(self):
-        spider = self.spider_class('example.com')
-        crawler = get_crawler()
-        with warnings.catch_warnings(record=True) as w:
-            spider.set_crawler(crawler)
-            self.assertIn("set_crawler", str(w[0].message))
-            self.assertTrue(hasattr(spider, 'crawler'))
-            self.assertIs(spider.crawler, crawler)
-            self.assertTrue(hasattr(spider, 'settings'))
-            self.assertIs(spider.settings, crawler.settings)
-
     def test_from_crawler_crawler_and_settings_population(self):
         crawler = get_crawler()
         spider = self.spider_class.from_crawler(crawler, 'example.com')
@@ -377,20 +366,6 @@ class CrawlSpiderTest(SpiderTest):
         self.assertTrue(hasattr(spider, '_follow_links'))
         self.assertFalse(spider._follow_links)
 
-    def test_follow_links_attribute_deprecated_population(self):
-        spider = self.spider_class('example.com')
-        self.assertFalse(hasattr(spider, '_follow_links'))
-
-        spider.set_crawler(get_crawler())
-        self.assertTrue(hasattr(spider, '_follow_links'))
-        self.assertTrue(spider._follow_links)
-
-        spider = self.spider_class('example.com')
-        settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False}
-        spider.set_crawler(get_crawler(settings_dict=settings_dict))
-        self.assertTrue(hasattr(spider, '_follow_links'))
-        self.assertFalse(spider._follow_links)
-
 
 class SitemapSpiderTest(SpiderTest):
 
@@ -578,57 +553,9 @@ Sitemap: /sitemap-relative-url.xml
 
 class DeprecationTest(unittest.TestCase):
 
-    def test_basespider_is_deprecated(self):
-        with warnings.catch_warnings(record=True) as w:
-
-            class MySpider1(BaseSpider):
-                pass
-
-            self.assertEqual(len(w), 1)
-            self.assertEqual(w[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(w[0].lineno, inspect.getsourcelines(MySpider1)[1])
-
-    def test_basespider_issubclass(self):
-        class MySpider2(Spider):
-            pass
-
-        class MySpider2a(MySpider2):
-            pass
-
-        class Foo(object):
-            pass
-
-        class Foo2(object_ref):
-            pass
-
-        assert issubclass(MySpider2, BaseSpider)
-        assert issubclass(MySpider2a, BaseSpider)
-        assert not issubclass(Foo, BaseSpider)
-        assert not issubclass(Foo2, BaseSpider)
-
-    def test_basespider_isinstance(self):
-        class MySpider3(Spider):
-            name = 'myspider3'
-
-        class MySpider3a(MySpider3):
-            pass
-
-        class Foo(object):
-            pass
-
-        class Foo2(object_ref):
-            pass
-
-        assert isinstance(MySpider3(), BaseSpider)
-        assert isinstance(MySpider3a(), BaseSpider)
-        assert not isinstance(Foo(), BaseSpider)
-        assert not isinstance(Foo2(), BaseSpider)
-
     def test_crawl_spider(self):
         assert issubclass(CrawlSpider, Spider)
-        assert issubclass(CrawlSpider, BaseSpider)
         assert isinstance(CrawlSpider(name='foo'), Spider)
-        assert isinstance(CrawlSpider(name='foo'), BaseSpider)
 
     def test_make_requests_from_url_deprecated(self):
         class MySpider4(Spider):

tests/test_utils_misc/__init__.py

@@ -3,12 +3,13 @@ import os
 import unittest
 
 from scrapy.item import Item, Field
-from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules
+from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
 
 from tests import mock
 
 __doctests__ = ['scrapy.utils.misc']
 
+
 class UtilsMiscTestCase(unittest.TestCase):
 
     def test_load_object(self):
@@ -130,5 +131,18 @@ class UtilsMiscTestCase(unittest.TestCase):
         with self.assertRaises(ValueError):
             create_instance(m, None, None)
 
+    def test_set_environ(self):
+        assert os.environ.get('some_test_environ') is None
+        with set_environ(some_test_environ='test_value'):
+            assert os.environ.get('some_test_environ') == 'test_value'
+        assert os.environ.get('some_test_environ') is None
+
+        os.environ['some_test_environ'] = 'test'
+        assert os.environ.get('some_test_environ') == 'test'
+        with set_environ(some_test_environ='test_value'):
+            assert os.environ.get('some_test_environ') == 'test_value'
+        assert os.environ.get('some_test_environ') == 'test'
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_utils_reqser.py

@@ -1,9 +1,12 @@
 # -*- coding: utf-8 -*-
 import unittest
+import sys
+
+import six
 
 from scrapy.http import Request, FormRequest
 from scrapy.spiders import Spider
-from scrapy.utils.reqser import request_to_dict, request_from_dict
+from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
 
 
 class RequestSerializationTest(unittest.TestCase):
@@ -26,6 +29,7 @@ class RequestSerializationTest(unittest.TestCase):
             encoding='latin-1',
             priority=20,
             meta={'a': 'b'},
+            cb_kwargs={'k': 'v'},
             flags=['testFlag'])
         self._assert_serializes_ok(r, spider=self.spider)
 
@@ -52,6 +56,7 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertEqual(r1.headers, r2.headers)
         self.assertEqual(r1.cookies, r2.cookies)
         self.assertEqual(r1.meta, r2.meta)
+        self.assertEqual(r1.cb_kwargs, r2.cb_kwargs)
         self.assertEqual(r1._encoding, r2._encoding)
         self.assertEqual(r1.priority, r2.priority)
         self.assertEqual(r1.dont_filter, r2.dont_filter)
@@ -68,6 +73,56 @@ class RequestSerializationTest(unittest.TestCase):
                     errback=self.spider.handle_error)
         self._assert_serializes_ok(r, spider=self.spider)
 
+    def test_private_callback_serialization(self):
+        r = Request("http://www.example.com",
+                    callback=self.spider._TestSpider__parse_item_private,
+                    errback=self.spider.handle_error)
+        self._assert_serializes_ok(r, spider=self.spider)
+
+    def test_mixin_private_callback_serialization(self):
+        if sys.version_info[0] < 3:
+            return
+        r = Request("http://www.example.com",
+                    callback=self.spider._TestSpiderMixin__mixin_callback,
+                    errback=self.spider.handle_error)
+        self._assert_serializes_ok(r, spider=self.spider)
+
+    def test_private_callback_name_matching(self):
+        self.assertTrue(_is_private_method('__a'))
+        self.assertTrue(_is_private_method('__a_'))
+        self.assertTrue(_is_private_method('__a_a'))
+        self.assertTrue(_is_private_method('__a_a_'))
+        self.assertTrue(_is_private_method('__a__a'))
+        self.assertTrue(_is_private_method('__a__a_'))
+        self.assertTrue(_is_private_method('__a___a'))
+        self.assertTrue(_is_private_method('__a___a_'))
+        self.assertTrue(_is_private_method('___a'))
+        self.assertTrue(_is_private_method('___a_'))
+        self.assertTrue(_is_private_method('___a_a'))
+        self.assertTrue(_is_private_method('___a_a_'))
+        self.assertTrue(_is_private_method('____a_a_'))
+
+        self.assertFalse(_is_private_method('_a'))
+        self.assertFalse(_is_private_method('_a_'))
+        self.assertFalse(_is_private_method('__a__'))
+        self.assertFalse(_is_private_method('__'))
+        self.assertFalse(_is_private_method('___'))
+        self.assertFalse(_is_private_method('____'))
+
+    def _assert_mangles_to(self, obj, name):
+        func = getattr(obj, name)
+        self.assertEqual(
+            _mangle_private_name(obj, func, func.__name__),
+            name
+        )
+
+    def test_private_name_mangling(self):
+        self._assert_mangles_to(
+            self.spider, '_TestSpider__parse_item_private')
+        if sys.version_info[0] >= 3:
+            self._assert_mangles_to(
+                self.spider, '_TestSpiderMixin__mixin_callback')
+
     def test_unserializable_callback1(self):
         r = Request("http://www.example.com", callback=lambda x: x)
         self.assertRaises(ValueError, request_to_dict, r)
@@ -78,7 +133,12 @@ class RequestSerializationTest(unittest.TestCase):
         self.assertRaises(ValueError, request_to_dict, r)
 
 
-class TestSpider(Spider):
+class TestSpiderMixin(object):
+    def __mixin_callback(self, response):
+        pass
+
+
+class TestSpider(Spider, TestSpiderMixin):
     name = 'test'
 
     def parse_item(self, response):
@@ -87,6 +147,9 @@ class TestSpider(Spider):
     def handle_error(self, failure):
         pass
 
+    def __parse_item_private(self, response):
+        pass
+
 
 class CustomRequest(Request):
     pass

tox.ini

@@ -105,6 +105,12 @@ deps = {[docs]deps}
 commands =
     sphinx-build -W -b html . {envtmpdir}/html
 
+[testenv:docs-coverage]
+changedir = {[docs]changedir}
+deps = {[docs]deps}
+commands =
+    sphinx-build -b coverage . {envtmpdir}/coverage
+
 [testenv:docs-links]
 changedir = {[docs]changedir}
 deps = {[docs]deps}