Merge remote-tracking branch 'upstream/master' into scheduler-refactoring-3

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.6.0
+current_version = 1.7.0
 commit = True
 tag = True
 tag_name = {new_version}

.travis.yml

@@ -1,4 +1,5 @@
 language: python
+dist: xenial
 branches:
   only:
     - master
@@ -6,26 +7,28 @@ branches:
     - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
 matrix:
   include:
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=jessie
-    - python: 2.7
-      env: TOXENV=pypy
-    - python: 2.7
-      env: TOXENV=pypy3
-    - python: 3.4
-      env: TOXENV=py34
-    - python: 3.5
-      env: TOXENV=py35
-    - python: 3.6
-      env: TOXENV=py36
-    - python: 3.7
-      env: TOXENV=py37
-      dist: xenial
-      sudo: true
-    - python: 3.6
-      env: TOXENV=docs
+    - env: TOXENV=py27
+      python: 2.7
+    - env: TOXENV=py27-pinned
+      python: 2.7
+    - env: TOXENV=py27-extra-deps
+      python: 2.7
+    - env: TOXENV=pypy
+      python: 2.7
+    - env: TOXENV=pypy3
+      python: 3.5
+    - env: TOXENV=py35
+      python: 3.5
+    - env: TOXENV=py35-pinned
+      python: 3.5
+    - env: TOXENV=py36
+      python: 3.6
+    - env: TOXENV=py37
+      python: 3.7
+    - env: TOXENV=py37-extra-deps
+      python: 3.7
+    - env: TOXENV=docs
+      python: 3.6
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then

README.rst

@@ -40,7 +40,7 @@ https://scrapy.org
 Requirements
 ============
 
-* Python 2.7 or Python 3.4+
+* Python 2.7 or Python 3.5+
 * Works on Linux, Windows, Mac OSX, BSD
 
 Install

docs/conf.py

@@ -30,6 +30,7 @@ extensions = [
     'scrapydocs',
     'sphinx.ext.autodoc',
     'sphinx.ext.coverage',
+    'sphinx.ext.intersphinx',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -74,6 +75,8 @@ language = 'en'
 # List of documents that shouldn't be included in the build.
 #unused_docs = []
 
+exclude_patterns = ['build']
+
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
 exclude_trees = ['.build']
@@ -250,3 +253,11 @@ coverage_ignore_pyobjects = [
     # Private exception used by the command-line interface implementation.
     r'^scrapy\.exceptions\.UsageError',
 ]
+
+
+# Options for the InterSphinx extension
+# -------------------------------------
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+}

docs/contributing.rst

@@ -171,6 +171,8 @@ Scrapy:
   See https://help.github.com/articles/setting-your-username-in-git/ for
   setup instructions.
 
+.. _documentation-policies:
+
 Documentation policies
 ======================
 
@@ -196,6 +198,8 @@ Tests
 Tests are implemented using the `Twisted unit-testing framework`_, running
 tests requires `tox`_.
 
+.. _running-tests:
+
 Running tests
 -------------
 

docs/faq.rst

@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy is supported under Python 2.7 and Python 3.4+
+Scrapy is supported under Python 2.7 and Python 3.5+
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 Python 2.6 support was dropped starting at Scrapy 0.20.
 Python 3 support was added in Scrapy 1.1.
@@ -329,6 +329,8 @@ I'm scraping a XML document and my XPath selector doesn't return any items
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 
+.. _faq-split-item:
+
 How to split an item into multiple items in an item pipeline?
 -------------------------------------------------------------
 

docs/intro/install.rst

@@ -7,7 +7,7 @@ Installation guide
 Installing Scrapy
 =================
 
-Scrapy runs on Python 2.7 and Python 3.4 or above
+Scrapy runs on Python 2.7 and Python 3.5 or above
 under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
 
 If you're using `Anaconda`_ or `Miniconda`_, you can install the package from

docs/news.rst

@@ -3,6 +3,346 @@
 Release notes
 =============
 
+.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
+          planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
+
+Scrapy 1.7.3 (2019-08-01)
+-------------------------
+
+Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
+
+Scrapy 1.7.2 (2019-07-23)
+-------------------------
+
+Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
+
+
+Scrapy 1.7.1 (2019-07-18)
+-------------------------
+
+Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
+
+.. _release-1.7.0:
+
+Scrapy 1.7.0 (2019-07-18)
+-------------------------
+
+.. note:: Make sure you install Scrapy 1.7.1. The Scrapy 1.7.0 package in PyPI
+          is the result of an erroneous commit tagging and does not include all
+          the changes described below.
+
+Highlights:
+
+* Improvements for crawls targeting multiple domains
+* A cleaner way to pass arguments to callbacks
+* A new class for JSON requests
+* Improvements for rule-based spiders
+* New features for feed exports
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+*   ``429`` is now part of the :setting:`RETRY_HTTP_CODES` setting by default
+
+    This change is **backward incompatible**. If you don’t want to retry
+    ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
+
+*   :class:`~scrapy.crawler.Crawler`,
+    :class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
+    :class:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
+    no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
+    only accept a :class:`~scrapy.spiders.Spider` subclass now.
+
+    :class:`~scrapy.spiders.Spider` subclass instances were never meant to
+    work, and they were not working as one would expect: instead of using the
+    passed :class:`~scrapy.spiders.Spider` subclass instance, their
+    :class:`~scrapy.spiders.Spider.from_crawler` method was called to generate
+    a new instance.
+
+*   Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting
+    may stop working. Scheduler priority queue classes now need to handle
+    :class:`~scrapy.http.Request` objects instead of arbitrary Python data
+    structures.
+
+See also :ref:`1.7-deprecation-removals` below.
+
+
+New features
+~~~~~~~~~~~~
+
+*   A new scheduler priority queue,
+    :class:`scrapy.pqueues.DownloaderAwarePriorityQueue`, may be
+    :ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
+    scheduling improvement on crawls targetting multiple web domains, at the
+    cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
+
+*   A new :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>` attribute
+    provides a cleaner way to pass keyword arguments to callback methods
+    (:issue:`1138`, :issue:`3563`)
+
+*   A new :class:`~scrapy.http.JSONRequest` class offers a more convenient way
+    to build JSON requests (:issue:`3504`, :issue:`3505`)
+
+*   A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
+    constructor now receives the :class:`~scrapy.http.Response` object that
+    originated the request as its second argument (:issue:`3682`)
+
+*   A new ``restrict_text`` parameter for the
+    :attr:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
+    constructor allows filtering links by linking text (:issue:`3622`,
+    :issue:`3635`)
+
+*   A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
+    for feeds exported to Amazon S3 (:issue:`3607`)
+
+*   A new :setting:`FEED_STORAGE_FTP_ACTIVE` setting allows using FTP’s active
+    connection mode for feeds exported to FTP servers (:issue:`3829`)
+
+*   A new :setting:`METAREFRESH_IGNORE_TAGS` setting allows overriding which
+    HTML tags are ignored when searching a response for HTML meta tags that
+    trigger a redirect (:issue:`1422`, :issue:`3768`)
+
+*   A new :reqmeta:`redirect_reasons` request meta key exposes the reason
+    (status code, meta refresh) behind every followed redirect (:issue:`3581`,
+    :issue:`3687`)
+
+*   The ``SCRAPY_CHECK`` variable is now set to the ``true`` string during runs
+    of the :command:`check` command, which allows :ref:`detecting contract
+    check runs from code <detecting-contract-check-runs>` (:issue:`3704`,
+    :issue:`3739`)
+
+*   A new :meth:`Item.deepcopy() <scrapy.item.Item.deepcopy>` method makes it
+    easier to :ref:`deep-copy items <copying-items>` (:issue:`1493`,
+    :issue:`3671`)
+
+*   :class:`~scrapy.extensions.corestats.CoreStats` also logs
+    ``elapsed_time_seconds`` now (:issue:`3638`)
+
+*   Exceptions from :class:`~scrapy.loader.ItemLoader` :ref:`input and output
+    processors <topics-loaders-processors>` are now more verbose
+    (:issue:`3836`, :issue:`3840`)
+
+*   :class:`~scrapy.crawler.Crawler`,
+    :class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
+    :class:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
+    now fail gracefully if they receive a :class:`~scrapy.spiders.Spider`
+    subclass instance instead of the subclass itself (:issue:`2283`,
+    :issue:`3610`, :issue:`3872`)
+
+
+Bug fixes
+~~~~~~~~~
+
+*   :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
+    is now also invoked for generators (:issue:`220`, :issue:`2061`)
+
+*   System exceptions like KeyboardInterrupt_ are no longer caught
+    (:issue:`3726`)
+
+*   :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` no
+    longer makes later calls to :meth:`ItemLoader.get_output_value()
+    <scrapy.loader.ItemLoader.get_output_value>` or
+    :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` return
+    empty data (:issue:`3804`, :issue:`3819`)
+
+*   The images pipeline (:class:`~scrapy.pipelines.images.ImagesPipeline`) no
+    longer ignores these Amazon S3 settings: :setting:`AWS_ENDPOINT_URL`,
+    :setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY`
+    (:issue:`3625`)
+
+*   Fixed a memory leak in :class:`~scrapy.pipelines.media.MediaPipeline`
+    affecting, for example, non-200 responses and exceptions from custom
+    middlewares (:issue:`3813`)
+
+*   Requests with private callbacks are now correctly unserialized from disk
+    (:issue:`3790`)
+
+*   :meth:`FormRequest.from_response() <scrapy.http.FormRequest.from_response>`
+    now handles invalid methods like major web browsers (:issue:`3777`,
+    :issue:`3794`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+*   A new topic, :ref:`topics-dynamic-content`, covers recommended approaches
+    to read dynamically-loaded data (:issue:`3703`)
+
+*   :ref:`topics-broad-crawls` now features information about memory usage
+    (:issue:`1264`, :issue:`3866`)
+
+*   The documentation of :class:`~scrapy.spiders.Rule` now covers how to access
+    the text of a link when using :class:`~scrapy.spiders.CrawlSpider`
+    (:issue:`3711`, :issue:`3712`)
+
+*   A new section, :ref:`httpcache-storage-custom`, covers writing a custom
+    cache storage backend for
+    :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
+    (:issue:`3683`, :issue:`3692`)
+
+*   A new :ref:`FAQ <faq>` entry, :ref:`faq-split-item`, explains what to do
+    when you want to split an item into multiple items from an item pipeline
+    (:issue:`2240`, :issue:`3672`)
+
+*   Updated the :ref:`FAQ entry about crawl order <faq-bfo-dfo>` to explain why
+    the first few requests rarely follow the desired order (:issue:`1739`,
+    :issue:`3621`)
+
+*   The :setting:`LOGSTATS_INTERVAL` setting (:issue:`3730`), the
+    :meth:`FilesPipeline.file_path <scrapy.pipelines.files.FilesPipeline.file_path>`
+    and
+    :meth:`ImagesPipeline.file_path <scrapy.pipelines.images.ImagesPipeline.file_path>`
+    methods (:issue:`2253`, :issue:`3609`) and the
+    :meth:`Crawler.stop() <scrapy.crawler.Crawler.stop>` method (:issue:`3842`)
+    are now documented
+
+*   Some parts of the documentation that were confusing or misleading are now
+    clearer (:issue:`1347`, :issue:`1789`, :issue:`2289`, :issue:`3069`,
+    :issue:`3615`, :issue:`3626`, :issue:`3668`, :issue:`3670`, :issue:`3673`,
+    :issue:`3728`, :issue:`3762`, :issue:`3861`, :issue:`3882`)
+
+*   Minor documentation fixes (:issue:`3648`, :issue:`3649`, :issue:`3662`,
+    :issue:`3674`, :issue:`3676`, :issue:`3694`, :issue:`3724`, :issue:`3764`,
+    :issue:`3767`, :issue:`3791`, :issue:`3797`, :issue:`3806`, :issue:`3812`)
+
+.. _1.7-deprecation-removals:
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+The following deprecated APIs have been removed (:issue:`3578`):
+
+*   ``scrapy.conf`` (use :attr:`Crawler.settings
+    <scrapy.crawler.Crawler.settings>`)
+
+*   From ``scrapy.core.downloader.handlers``:
+
+    *   ``http.HttpDownloadHandler`` (use ``http10.HTTP10DownloadHandler``)
+
+*   ``scrapy.loader.ItemLoader._get_values`` (use ``_get_xpathvalues``)
+
+*   ``scrapy.loader.XPathItemLoader`` (use :class:`~scrapy.loader.ItemLoader`)
+
+*   ``scrapy.log`` (see :ref:`topics-logging`)
+
+*   From ``scrapy.pipelines``:
+
+    *   ``files.FilesPipeline.file_key`` (use ``file_path``)
+
+    *   ``images.ImagesPipeline.file_key`` (use ``file_path``)
+
+    *   ``images.ImagesPipeline.image_key`` (use ``file_path``)
+
+    *   ``images.ImagesPipeline.thumb_key`` (use ``thumb_path``)
+
+*   From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``:
+
+    *   ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+    *   ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+    *   ``XPathSelector`` (use :class:`~scrapy.selector.Selector`)
+
+    *   ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`)
+
+*   From ``scrapy.selector.csstranslator``:
+
+    *   ``ScrapyGenericTranslator`` (use parsel.csstranslator.GenericTranslator_)
+
+    *   ``ScrapyHTMLTranslator`` (use parsel.csstranslator.HTMLTranslator_)
+
+    *   ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_)
+
+*   From :class:`~scrapy.selector.Selector`:
+
+    *   ``_root`` (both the constructor argument and the object property, use
+        ``root``)
+
+    *   ``extract_unquoted`` (use ``getall``)
+
+    *   ``select`` (use ``xpath``)
+
+*   From :class:`~scrapy.selector.SelectorList`:
+
+    *   ``extract_unquoted`` (use ``getall``)
+
+    *   ``select`` (use ``xpath``)
+
+    *   ``x`` (use ``xpath``)
+
+*   ``scrapy.spiders.BaseSpider`` (use :class:`~scrapy.spiders.Spider`)
+
+*   From :class:`~scrapy.spiders.Spider` (and subclasses):
+
+    *   ``DOWNLOAD_DELAY`` (use :ref:`download_delay
+        <spider-download_delay-attribute>`)
+
+    *   ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`)
+
+*   ``scrapy.spiders.spiders`` (use :class:`~scrapy.spiderloader.SpiderLoader`)
+
+*   ``scrapy.telnet`` (use :mod:`scrapy.extensions.telnet`)
+
+*   From ``scrapy.utils.python``:
+
+    *   ``str_to_unicode`` (use ``to_unicode``)
+
+    *   ``unicode_to_str`` (use ``to_bytes``)
+
+*   ``scrapy.utils.response.body_or_str``
+
+The following deprecated settings have also been removed (:issue:`3578`):
+
+*   ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+*   The ``queuelib.PriorityQueue`` value for the
+    :setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use
+    :class:`scrapy.pqueues.ScrapyPriorityQueue` instead.
+
+*   ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that
+    do not accept two arguments are deprecated.
+
+*   The following modules are deprecated:
+
+    *   ``scrapy.utils.http`` (use `w3lib.http`_)
+
+    *   ``scrapy.utils.markup`` (use `w3lib.html`_)
+
+    *   ``scrapy.utils.multipart`` (use `urllib3`_)
+
+*   The ``scrapy.utils.datatypes.MergeDict`` class is deprecated for Python 3
+    code bases. Use :class:`~collections.ChainMap` instead. (:issue:`3878`)
+
+*   The ``scrapy.utils.gz.is_gzipped`` function is deprecated. Use
+    ``scrapy.utils.gz.gzip_magic_number`` instead.
+
+.. _urllib3: https://urllib3.readthedocs.io/en/latest/index.html
+.. _w3lib.html: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.html
+.. _w3lib.http: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.http
+
+
+Other changes
+~~~~~~~~~~~~~
+
+*   It is now possible to run all tests from the same tox_ environment in
+    parallel; the documentation now covers :ref:`this and other ways to run
+    tests <running-tests>` (:issue:`3707`)
+
+*   It is now possible to generate an API documentation coverage report
+    (:issue:`3806`, :issue:`3810`, :issue:`3860`)
+
+*   The :ref:`documentation policies <documentation-policies>` now require
+    docstrings_ (:issue:`3701`) that follow `PEP 257`_ (:issue:`3748`)
+
+*   Internal fixes and cleanup (:issue:`3629`, :issue:`3643`, :issue:`3684`,
+    :issue:`3698`, :issue:`3734`, :issue:`3735`, :issue:`3736`, :issue:`3737`,
+    :issue:`3809`, :issue:`3821`, :issue:`3825`, :issue:`3827`, :issue:`3833`,
+    :issue:`3857`, :issue:`3877`)
+
 .. _release-1.6.0:
 
 Scrapy 1.6.0 (2019-01-30)
@@ -2471,12 +2811,22 @@ First release of Scrapy.
 
 .. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
 .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
-.. _w3lib: https://github.com/scrapy/w3lib
-.. _scrapely: https://github.com/scrapy/scrapely
-.. _marshal: https://docs.python.org/2/library/marshal.html
-.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
-.. _lxml: http://lxml.de/
 .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
-.. _resource: https://docs.python.org/2/library/resource.html
+.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
+.. _cssselect: https://github.com/scrapy/cssselect/
+.. _docstrings: https://docs.python.org/glossary.html#term-docstring
+.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
+.. _lxml: http://lxml.de/
+.. _marshal: https://docs.python.org/2/library/marshal.html
+.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
+.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
+.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
+.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
 .. _queuelib: https://github.com/scrapy/queuelib
-.. _cssselect: https://github.com/SimonSapin/cssselect
+.. _resource: https://docs.python.org/2/library/resource.html
+.. _scrapely: https://github.com/scrapy/scrapely
+.. _tox: https://pypi.python.org/pypi/tox
+.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
+.. _w3lib: https://github.com/scrapy/w3lib
+.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
+.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1

docs/topics/contracts.rst

@@ -126,6 +126,7 @@ response received::
                 if header not in response.headers:
                     raise ContractFail('X-CustomHeader not present')
 
+.. _detecting-contract-check-runs:
 
 Detecting check runs
 ====================

docs/topics/downloader-middleware.rst

@@ -52,6 +52,8 @@ as its value.  For example, if you want to disable the user-agent middleware::
 Finally, keep in mind that some middlewares may need to be enabled through a
 particular setting. See each middleware documentation for more info.
 
+.. _topics-downloader-middleware-custom:
+
 Writing your own downloader middleware
 ======================================
 
@@ -961,7 +963,7 @@ precedence over the :setting:`RETRY_TIMES` setting.
 RETRY_HTTP_CODES
 ^^^^^^^^^^^^^^^^
 
-Default: ``[500, 502, 503, 504, 522, 524, 408]``
+Default: ``[500, 502, 503, 504, 522, 524, 408, 429]``
 
 Which HTTP response codes to retry. Other errors (DNS lookup issues,
 connections lost, etc) are always retried.
@@ -987,6 +989,17 @@ RobotsTxtMiddleware
     To make sure Scrapy respects robots.txt make sure the middleware is enabled
     and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
 
+    This middleware has to be combined with a robots.txt_ parser.
+
+    Scrapy ships with support for the following robots.txt_ parsers:
+
+    * :ref:`RobotFileParser <python-robotfileparser>` (default)
+    * :ref:`Reppy <reppy-parser>`
+    * :ref:`Robotexclusionrulesparser <rerp-parser>`
+
+    You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+    setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
+
 .. reqmeta:: dont_obey_robotstxt
 
 If :attr:`Request.meta <scrapy.http.Request.meta>` has
@@ -994,6 +1007,74 @@ If :attr:`Request.meta <scrapy.http.Request.meta>` has
 the request will be ignored by this middleware even if
 :setting:`ROBOTSTXT_OBEY` is enabled.
 
+.. _python-robotfileparser:
+
+RobotFileParser
+~~~~~~~~~~~~~~~
+
+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ is 
+Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's 
+1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_. It lacks
+support for wildcard matching. Scrapy uses this parser by default.
+
+In order to use this parser, set:
+
+* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser``
+
+.. _rerp-parser:
+
+Robotexclusionrulesparser
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ is fully compliant
+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
+with support for wildcard matching.
+
+In order to use this parser:
+
+* Install `Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_ by running
+  ``pip install robotexclusionrulesparser``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.RerpRobotParser``
+
+.. _reppy-parser:
+
+Reppy parser
+~~~~~~~~~~~~
+
+`Reppy <https://github.com/seomoz/reppy/>`_ is a Python wrapper around `Robots Exclusion
+Protocol Parser for C++ <https://github.com/seomoz/rep-cpp>`_. The parser is fully compliant
+with `Martijn Koster's 1996 draft specification <http://www.robotstxt.org/norobots-rfc.txt>`_,
+with support for wildcard matching. Unlike
+`RobotFileParser <https://docs.python.org/3.7/library/urllib.robotparser.html>`_ and
+`Robotexclusionrulesparser <http://nikitathespider.com/python/rerp/>`_, it uses the length based
+rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific
+rule based on the length of the path trumps the less specific (shorter) rule.
+
+In order to use this parser:
+
+* Install `Reppy <https://github.com/seomoz/reppy/>`_ by running ``pip install reppy``
+
+* Set :setting:`ROBOTSTXT_PARSER` setting to
+  ``scrapy.robotstxt.ReppyRobotParser``
+
+.. _support-for-new-robots-parser:
+
+Implementing support for a new parser
+-------------------------------------
+
+You can implement support for a new robots.txt_ parser by subclassing
+the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
+implementing the methods described below.
+
+.. module:: scrapy.robotstxt
+   :synopsis: robots.txt parser interface and implementations
+
+.. autoclass:: RobotParser
+   :members:
+
+.. _robots.txt: http://www.robotstxt.org/
 
 DownloaderStats
 ---------------

docs/topics/feed-exports.rst

@@ -164,9 +164,9 @@ The feeds are stored in a FTP server.
  * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
  * Required external libraries: none
 
-FTP supports two different connection modes: [active or passive](
-https://stackoverflow.com/a/1699163). Scrapy uses the passive connection mode
-by default. To use the active connection mode instead, set the
+FTP supports two different connection modes: `active or passive
+<https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
+mode by default. To use the active connection mode instead, set the
 :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 
 .. _topics-feed-storage-s3:
@@ -320,8 +320,11 @@ FEED_STORAGE_FTP_ACTIVE
 
 Default: ``False``
 
-Whether to use [active mode](https://stackoverflow.com/a/1699163) when exporting feeds
-to a FTP server.
+Whether to use the active connection mode when exporting feeds to an FTP server
+(``True``) or use the passive connection mode instead (``False``, default).
+
+For information about FTP connection modes, see `What is the difference between
+active and passive FTP? <https://stackoverflow.com/a/1699163>`_.
 
 .. setting:: FEED_STORAGE_S3_ACL
 

docs/topics/items.rst

@@ -157,6 +157,8 @@ To access all populated values, just use the typical `dict API`_::
     [('price', 1000), ('name', 'Desktop PC')]
 
 
+.. _copying-items:
+
 Copying items
 -------------
 

docs/topics/logging.rst

@@ -193,6 +193,17 @@ to override some of the Scrapy settings regarding logging.
     Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
         Further documentation on available handlers
 
+.. _custom-log-formats:
+
+Custom Log Formats
+------------------
+
+A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
+and making :setting:`LOG_FORMATTER` point to your new class.
+ 
+.. autoclass:: scrapy.logformatter.LogFormatter
+   :members:
+
 Advanced customization
 ----------------------
 

docs/topics/practices.rst

@@ -34,15 +34,16 @@ Here's an example showing how to run a single spider with it.
         # Your spider definition
         ...
 
-    process = CrawlerProcess({
-        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'
+    process = CrawlerProcess(settings={
+        'FEED_FORMAT': 'json',
+        'FEED_URI': 'items.json'
     })
 
     process.crawl(MySpider)
     process.start() # the script will block here until the crawling is finished
 
-Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get
-acquainted with its usage details.
+Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess`
+documentation to get acquainted with its usage details.
 
 If you are inside a Scrapy project there are some additional helpers you can
 use to import those components within the project. You can automatically import

docs/topics/settings.rst

@@ -30,6 +30,8 @@ Python `import search path`_.
 
 .. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
 
+.. _populating-settings:
+
 Populating the settings
 =======================
 
@@ -438,9 +440,10 @@ or even enable client-side authentication (and various other things).
     which uses the platform's certificates to validate remote endpoints.
     **This is only available if you use Twisted>=14.0.**
 
-If you do use a custom ContextFactory, make sure it accepts a ``method``
-parameter at init (this is the ``OpenSSL.SSL`` method mapping
-:setting:`DOWNLOADER_CLIENT_TLS_METHOD`).
+If you do use a custom ContextFactory, make sure its ``__init__`` method
+accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
+:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``tls_verbose_logging``
+parameter (``bool``).
 
 .. setting:: DOWNLOADER_CLIENT_TLS_METHOD
 
@@ -468,6 +471,20 @@ This setting must be one of these string values:
     We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
     or above (Twisted>=14.0 if you can).
 
+.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
+-------------------------------------
+
+Default: ``False``
+
+Setting this to ``True`` will enable DEBUG level messages about TLS connection
+parameters after establishing HTTPS connections. The kind of information logged
+depends on the versions of OpenSSL and pyOpenSSL.
+
+This setting is only used for the default
+:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
+
 .. setting:: DOWNLOADER_MIDDLEWARES
 
 DOWNLOADER_MIDDLEWARES
@@ -538,6 +555,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti
 When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
 per ip address instead of per domain.
 
+.. _spider-download_delay-attribute:
+
 You can also change this setting per spider by setting ``download_delay``
 spider attribute.
 
@@ -866,6 +885,15 @@ directives.
 
 .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
 
+.. setting:: LOG_FORMATTER
+
+LOG_FORMATTER
+-------------
+
+Default: :class:`scrapy.logformatter.LogFormatter`
+
+The class to use for :ref:`formatting log messages <custom-log-formats>` for different actions.
+
 .. setting:: LOG_LEVEL
 
 LOG_LEVEL
@@ -1113,6 +1141,16 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
     this option is enabled by default in settings.py file generated
     by ``scrapy startproject`` command.
 
+.. setting:: ROBOTSTXT_PARSER
+
+ROBOTSTXT_PARSER
+----------------
+
+Default: ``'scrapy.robotstxt.PythonRobotParser'``
+
+The parser backend to use for parsing ``robots.txt`` files. For more information see
+:ref:`topics-dlmw-robots`.
+
 .. setting:: SCHEDULER
 
 SCHEDULER

requirements-py2.txt

@@ -1,10 +1,17 @@
-Twisted>=13.1.0
-lxml
-pyOpenSSL
-cssselect>=0.9
-queuelib
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
 PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+w3lib>=1.17.0
+
+pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0  # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+Twisted>=16.0.0
+zope.interface>=4.1.3

requirements-py3.txt

@@ -1,10 +1,17 @@
-Twisted>=17.9.0
-lxml>=3.2.4
-pyOpenSSL>=0.13.1
-cssselect>=0.9
-queuelib>=1.1.1
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
 PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+Twisted>=17.9.0
+w3lib>=1.17.0
+
+pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0  # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+zope.interface>=4.1.3

scrapy/VERSION

@@ -1 +1 @@
-1.6.0
+1.7.0

scrapy/_monkeypatches.py

@@ -1,14 +1,10 @@
-import sys
+import six
 from six.moves import copyreg
 
-if sys.version_info[0] == 2:
+
+if six.PY2:
     from urlparse import urlparse
-
-    # workaround for https://bugs.python.org/issue7904 - Python < 2.7
-    if urlparse('s3://bucket/key').netloc != 'bucket':
-        from urlparse import uses_netloc
-        uses_netloc.append('s3')
-
+    
     # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
     if urlparse('s3://bucket/key?key=value').query != 'key=value':
         from urlparse import uses_query

scrapy/core/downloader/contextfactory.py

@@ -28,9 +28,15 @@ if twisted_version >= (14, 0, 0):
          understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
         """
 
-        def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
+        def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
             super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
             self._ssl_method = method
+            self.tls_verbose_logging = tls_verbose_logging
+
+        @classmethod
+        def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
+            tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
+            return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
 
         def getCertificateOptions(self):
             # setting verify=True will require you to provide CAs
@@ -56,7 +62,8 @@ if twisted_version >= (14, 0, 0):
             return self.getCertificateOptions().getContext()
 
         def creatorForNetloc(self, hostname, port):
-            return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
+            return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
+                                          verbose_logging=self.tls_verbose_logging)
 
 
     @implementer(IPolicyForHTTPS)

scrapy/core/downloader/handlers/ftp.py

@@ -59,7 +59,7 @@ class ReceivedDataProtocol(Protocol):
     def close(self):
         self.body.close() if self.filename else self.body.seek(0)
 
-_CODE_RE = re.compile("\d+")
+_CODE_RE = re.compile(r"\d+")
 
 
 class FTPDownloadHandler(object):

scrapy/core/downloader/handlers/http10.py

@@ -1,7 +1,7 @@
 """Download handlers for http and https schemes
 """
 from twisted.internet import reactor
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.python import to_unicode
 
 
@@ -11,6 +11,7 @@ class HTTP10DownloadHandler(object):
     def __init__(self, settings):
         self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
         self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+        self._settings = settings
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
@@ -21,7 +22,7 @@ class HTTP10DownloadHandler(object):
     def _connect(self, factory):
         host, port = to_unicode(factory.host), factory.port
         if factory.scheme == b'https':
-            return reactor.connectSSL(host, port, factory,
-                                      self.ClientContextFactory())
+            client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None)
+            return reactor.connectSSL(host, port, factory, client_context_factory)
         else:
             return reactor.connectTCP(host, port, factory)

scrapy/core/downloader/handlers/http11.py

@@ -25,7 +25,7 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.core.downloader.tls import openssl_methods
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import load_object, create_instance
 from scrapy.utils.python import to_bytes, to_unicode
 from scrapy import twisted_version
 
@@ -44,14 +44,15 @@ class HTTP11DownloadHandler(object):
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
         # try method-aware context factory
         try:
-            self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
+            self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None,
+                                                   method=self._sslMethod)
         except TypeError:
             # use context factory defaults
-            self._contextFactory = self._contextFactoryClass()
+            self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None)
             msg = """
  '%s' does not accept `method` argument (type OpenSSL.SSL method,\
- e.g. OpenSSL.SSL.SSLv23_METHOD).\
- Please upgrade your context factory class to handle it or ignore it.""" % (
+ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\
+ Please upgrade your context factory class to handle them or ignore them.""" % (
                 settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
             warnings.warn(msg)
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
@@ -101,7 +102,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     for it.
     """
 
-    _responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
+    _responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory,
                  timeout=30, bindAddress=None):
@@ -479,10 +480,10 @@ class _ResponseReader(protocol.Protocol):
                 return
 
             elif not self._fail_on_dataloss_warned:
-                logger.warn("Got data loss in %s. If you want to process broken "
-                            "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
-                            " -- This message won't be shown in further requests",
-                            self._txresponse.request.absoluteURI.decode())
+                logger.warning("Got data loss in %s. If you want to process broken "
+                               "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
+                               " -- This message won't be shown in further requests",
+                               self._txresponse.request.absoluteURI.decode())
                 self._fail_on_dataloss_warned = True
 
         self._finished.errback(reason)

scrapy/core/downloader/tls.py

@@ -2,6 +2,7 @@ import logging
 from OpenSSL import SSL
 
 from scrapy import twisted_version
+from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 
 
 logger = logging.getLogger(__name__)
@@ -20,6 +21,7 @@ openssl_methods = {
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
 }
 
+
 if twisted_version >= (14, 0, 0):
     # ClientTLSOptions requires a recent-enough version of Twisted.
     # Not having ScrapyClientTLSOptions should not matter for older
@@ -65,13 +67,39 @@ if twisted_version >= (14, 0, 0):
         Same as Twisted's private _sslverify.ClientTLSOptions,
         except that VerificationError, CertificateError and ValueError
         exceptions are caught, so that the connection is not closed, only
-        logging warnings.
+        logging warnings. Also, HTTPS connection parameters logging is added.
         """
 
+        def __init__(self, hostname, ctx, verbose_logging=False):
+            super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
+            self.verbose_logging = verbose_logging
+
         def _identityVerifyingInfoCallback(self, connection, where, ret):
             if where & SSL_CB_HANDSHAKE_START:
                 set_tlsext_host_name(connection, self._hostnameBytes)
             elif where & SSL_CB_HANDSHAKE_DONE:
+                if self.verbose_logging:
+                    if hasattr(connection, 'get_cipher_name'):  # requires pyOPenSSL 0.15
+                        if hasattr(connection, 'get_protocol_version_name'):  # requires pyOPenSSL 16.0.0
+                            logger.debug('SSL connection to %s using protocol %s, cipher %s',
+                                         self._hostnameASCII,
+                                         connection.get_protocol_version_name(),
+                                         connection.get_cipher_name(),
+                                         )
+                        else:
+                            logger.debug('SSL connection to %s using cipher %s',
+                                         self._hostnameASCII,
+                                         connection.get_cipher_name(),
+                                         )
+                    server_cert = connection.get_peer_certificate()
+                    logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
+                                 x509name_to_string(server_cert.get_issuer()),
+                                 x509name_to_string(server_cert.get_subject()),
+                                 )
+                    key_info = get_temp_key_info(connection._ssl)
+                    if key_info:
+                        logger.debug('SSL temp key: %s', key_info)
+
                 try:
                     verifyHostname(connection, self._hostnameASCII)
                 except verification_errors as e:

scrapy/core/downloader/webclient.py

@@ -95,7 +95,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
 
 class ScrapyHTTPClientFactory(HTTPClientFactory):
     """Scrapy implementation of the HTTPClientFactory overwriting the
-    serUrl method to make use of our Url object that cache the parse
+    setUrl method to make use of our Url object that cache the parse
     result.
     """
 

scrapy/crawler.py

@@ -39,14 +39,15 @@ class Crawler(object):
         self.settings = settings.copy()
         self.spidercls.update_settings(self.settings)
 
-        d = dict(overridden_settings(self.settings))
-        logger.info("Overridden settings: %(settings)r", {'settings': d})
-
         self.signals = SignalManager(self)
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
 
         handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
         logging.root.addHandler(handler)
+
+        d = dict(overridden_settings(self.settings))
+        logger.info("Overridden settings: %(settings)r", {'settings': d})
+
         if get_scrapy_root_handler() is not None:
             # scrapy root handler already installed: update it with new settings
             install_scrapy_root_handler(self.settings)

scrapy/downloadermiddlewares/robotstxt.py

@@ -5,8 +5,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 """
 
 import logging
-
-from six.moves.urllib import robotparser
+import sys
+import re
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import NotConfigured, IgnoreRequest
@@ -14,6 +14,7 @@ from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import to_native_str
+from scrapy.utils.misc import load_object
 
 logger = logging.getLogger(__name__)
 
@@ -24,10 +25,13 @@ class RobotsTxtMiddleware(object):
     def __init__(self, crawler):
         if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
             raise NotConfigured
-
+        self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy')
         self.crawler = crawler
-        self._useragent = crawler.settings.get('USER_AGENT')
         self._parsers = {}
+        self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER'))
+
+        # check if parser dependencies are met, this should throw an error otherwise.
+        self._parserimpl.from_crawler(self.crawler, b'')
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -43,7 +47,8 @@ class RobotsTxtMiddleware(object):
     def process_request_2(self, rp, request, spider):
         if rp is None:
             return
-        if not rp.can_fetch(to_native_str(self._useragent), request.url):
+        useragent = request.headers.get(b'User-Agent', self._default_useragent)
+        if not rp.allowed(request.url, useragent):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
             self.crawler.stats.inc_value('robotstxt/forbidden')
@@ -62,13 +67,14 @@ class RobotsTxtMiddleware(object):
                 meta={'dont_obey_robotstxt': True}
             )
             dfd = self.crawler.engine.download(robotsreq, spider)
-            dfd.addCallback(self._parse_robots, netloc)
+            dfd.addCallback(self._parse_robots, netloc, spider)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)
             self.crawler.stats.inc_value('robotstxt/request_count')
 
         if isinstance(self._parsers[netloc], Deferred):
             d = Deferred()
+
             def cb(result):
                 d.callback(result)
                 return result
@@ -85,27 +91,10 @@ class RobotsTxtMiddleware(object):
                          extra={'spider': spider})
         return failure
 
-    def _parse_robots(self, response, netloc):
+    def _parse_robots(self, response, netloc, spider):
         self.crawler.stats.inc_value('robotstxt/response_count')
-        self.crawler.stats.inc_value(
-            'robotstxt/response_status_count/{}'.format(response.status))
-        rp = robotparser.RobotFileParser(response.url)
-        body = ''
-        if hasattr(response, 'text'):
-            body = response.text
-        else:  # last effort try
-            try:
-                body = response.body.decode('utf-8')
-            except UnicodeDecodeError:
-                # If we found garbage, disregard it:,
-                # but keep the lookup cached (in self._parsers)
-                # Running rp.parse() will set rp state from
-                # 'disallow all' to 'allow any'.
-                self.crawler.stats.inc_value('robotstxt/unicode_error_count')
-        # stdlib's robotparser expects native 'str' ;
-        # with unicode input, non-ASCII encoded bytes decoding fails in Python2
-        rp.parse(to_native_str(body).splitlines())
-
+        self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status))
+        rp = self._parserimpl.from_crawler(self.crawler, response.body)
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = rp
         rp_dfd.callback(rp)

scrapy/item.py

@@ -4,16 +4,22 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
-from pprint import pformat
-from collections import MutableMapping
-from copy import deepcopy
-
 from abc import ABCMeta
+from pprint import pformat
+from copy import deepcopy
+import collections
+
 import six
 
 from scrapy.utils.trackref import object_ref
 
 
+if six.PY2:
+    MutableMapping = collections.MutableMapping
+else:
+    MutableMapping = collections.abc.MutableMapping
+
+
 class BaseItem(object_ref):
     """Base class for all scraped items."""
     pass

scrapy/loader/processors.py

@@ -3,10 +3,13 @@ This module provides some commonly used processors for Item Loaders.
 
 See documentation in docs/topics/loaders.rst
 """
+try:
+    from collections import ChainMap
+except ImportError:
+    from scrapy.utils.datatypes import MergeDict as ChainMap
 
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.datatypes import MergeDict
-from .common import wrap_loader_context
+from scrapy.loader.common import wrap_loader_context
 
 
 class MapCompose(object):
@@ -18,7 +21,7 @@ class MapCompose(object):
     def __call__(self, value, loader_context=None):
         values = arg_to_iter(value)
         if loader_context:
-            context = MergeDict(loader_context, self.default_loader_context)
+            context = ChainMap(loader_context, self.default_loader_context)
         else:
             context = self.default_loader_context
         wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
@@ -45,7 +48,7 @@ class Compose(object):
 
     def __call__(self, value, loader_context=None):
         if loader_context:
-            context = MergeDict(loader_context, self.default_loader_context)
+            context = ChainMap(loader_context, self.default_loader_context)
         else:
             context = self.default_loader_context
         wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]

scrapy/logformatter.py

@@ -12,26 +12,40 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref
 
 class LogFormatter(object):
     """Class for generating log messages for different actions.
-
-    All methods must return a dictionary listing the parameters ``level``,
-    ``msg`` and ``args`` which are going to be used for constructing the log
-    message when calling logging.log.
+    
+    All methods must return a dictionary listing the parameters ``level``, ``msg``
+    and ``args`` which are going to be used for constructing the log message when
+    calling ``logging.log``.
 
     Dictionary keys for the method outputs:
-        * ``level`` should be the log level for that action, you can use those
-        from the python logging library: logging.DEBUG, logging.INFO,
-        logging.WARNING, logging.ERROR and logging.CRITICAL.
 
-        * ``msg`` should be a string that can contain different formatting
-        placeholders. This string, formatted with the provided ``args``, is
-        going to be the log message for that action.
+    *   ``level`` is the log level for that action, you can use those from the
+        `python logging library <https://docs.python.org/3/library/logging.html>`_ :
+        ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR``
+        and ``logging.CRITICAL``.
+    *   ``msg`` should be a string that can contain different formatting placeholders.
+        This string, formatted with the provided ``args``, is going to be the long message
+        for that action.
+    *   ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
+        The final log message is computed as ``msg % args``.
 
-        * ``args`` should be a tuple or dict with the formatting placeholders
-        for ``msg``.  The final log message is computed as output['msg'] %
-        output['args'].
+    Here is an example on how to create a custom log formatter to lower the severity level of
+    the log message when an item is dropped from the pipeline::
+
+            class PoliteLogFormatter(logformatter.LogFormatter):
+                def dropped(self, item, exception, response, spider):
+                    return {
+                        'level': logging.INFO, # lowering the level from logging.WARNING
+                        'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
+                        'args': {
+                            'exception': exception,
+                            'item': item,
+                        }
+                    }
     """
-
+    
     def crawled(self, request, response, spider):
+        """Logs a message when the crawler finds a webpage."""
         request_flags = ' %s' % str(request.flags) if request.flags else ''
         response_flags = ' %s' % str(response.flags) if response.flags else ''
         return {
@@ -40,7 +54,7 @@ class LogFormatter(object):
             'args': {
                 'status': response.status,
                 'request': request,
-                'request_flags' : request_flags,
+                'request_flags': request_flags,
                 'referer': referer_str(request),
                 'response_flags': response_flags,
                 # backward compatibility with Scrapy logformatter below 1.4 version
@@ -49,6 +63,7 @@ class LogFormatter(object):
         }
 
     def scraped(self, item, response, spider):
+        """Logs a message when an item is scraped by a spider."""
         if isinstance(response, Failure):
             src = response.getErrorMessage()
         else:
@@ -63,6 +78,7 @@ class LogFormatter(object):
         }
 
     def dropped(self, item, exception, response, spider):
+        """Logs a message when an item is dropped while it is passing through the item pipeline."""
         return {
             'level': logging.WARNING,
             'msg': DROPPEDMSG,

scrapy/pipelines/files.py

@@ -189,6 +189,19 @@ class S3FilesStore(object):
             'X-Amz-Grant-Read': 'GrantRead',
             'X-Amz-Grant-Read-ACP': 'GrantReadACP',
             'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
+            'X-Amz-Object-Lock-Legal-Hold': 'ObjectLockLegalHoldStatus',
+            'X-Amz-Object-Lock-Mode': 'ObjectLockMode',
+            'X-Amz-Object-Lock-Retain-Until-Date': 'ObjectLockRetainUntilDate',
+            'X-Amz-Request-Payer': 'RequestPayer',
+            'X-Amz-Server-Side-Encryption': 'ServerSideEncryption',
+            'X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id': 'SSEKMSKeyId',
+            'X-Amz-Server-Side-Encryption-Context': 'SSEKMSEncryptionContext',
+            'X-Amz-Server-Side-Encryption-Customer-Algorithm': 'SSECustomerAlgorithm',
+            'X-Amz-Server-Side-Encryption-Customer-Key': 'SSECustomerKey',
+            'X-Amz-Server-Side-Encryption-Customer-Key-Md5': 'SSECustomerKeyMD5',
+            'X-Amz-Storage-Class': 'StorageClass',
+            'X-Amz-Tagging': 'Tagging',
+            'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
         })
         extra = {}
         for key, value in six.iteritems(headers):

scrapy/robotstxt.py

@@ -0,0 +1,112 @@
+import sys
+import logging
+from abc import ABCMeta, abstractmethod
+from six import with_metaclass
+
+from scrapy.utils.python import to_native_str, to_unicode
+
+logger = logging.getLogger(__name__)
+
+
+class RobotParser(with_metaclass(ABCMeta)):
+    @classmethod
+    @abstractmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        """Parse the content of a robots.txt_ file as bytes. This must be a class method.
+        It must return a new instance of the parser backend.
+
+        :param crawler: crawler which made the request
+        :type crawler: :class:`~scrapy.crawler.Crawler` instance
+
+        :param robotstxt_body: content of a robots.txt_ file.
+        :type robotstxt_body: bytes
+        """
+        pass
+
+    @abstractmethod
+    def allowed(self, url, user_agent):
+        """Return ``True`` if  ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
+
+        :param url: Absolute URL
+        :type url: string
+
+        :param user_agent: User agent
+        :type user_agent: string
+        """
+        pass
+
+
+class PythonRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from six.moves.urllib_robotparser import RobotFileParser
+        self.spider = spider
+        try:
+            robotstxt_body = to_native_str(robotstxt_body)
+        except UnicodeDecodeError:
+            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+            # Switch to 'allow all' state.
+            logger.warning("Failure while parsing robots.txt using %(parser)s."
+                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+                           {'parser': "RobotFileParser"},
+                           exc_info=sys.exc_info(),
+                           extra={'spider': self.spider})
+            robotstxt_body = ''
+        self.rp = RobotFileParser()
+        self.rp.parse(robotstxt_body.splitlines())
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        user_agent = to_native_str(user_agent)
+        url = to_native_str(url)
+        return self.rp.can_fetch(user_agent, url)
+
+
+class ReppyRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from reppy.robots import Robots
+        self.spider = spider
+        self.rp = Robots.parse('', robotstxt_body)
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        return self.rp.allowed(url, user_agent)
+
+
+class RerpRobotParser(RobotParser):
+    def __init__(self, robotstxt_body, spider):
+        from robotexclusionrulesparser import RobotExclusionRulesParser
+        self.spider = spider
+        self.rp = RobotExclusionRulesParser()
+        try:
+            robotstxt_body = robotstxt_body.decode('utf-8')
+        except UnicodeDecodeError:
+            # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
+            # Switch to 'allow all' state.
+            logger.warning("Failure while parsing robots.txt using %(parser)s."
+                           " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.",
+                           {'parser': "RobotExclusionRulesParser"},
+                           exc_info=sys.exc_info(),
+                           extra={'spider': self.spider})
+            robotstxt_body = ''
+        self.rp.parse(robotstxt_body)
+
+    @classmethod
+    def from_crawler(cls, crawler, robotstxt_body):
+        spider = None if not crawler else crawler.spider
+        o = cls(robotstxt_body, spider)
+        return o
+
+    def allowed(self, url, user_agent):
+        user_agent = to_unicode(user_agent)
+        url = to_unicode(url)
+        return self.rp.is_allowed(user_agent, url)

scrapy/settings/__init__.py

@@ -1,11 +1,17 @@
 import six
 import json
 import copy
-from collections import MutableMapping
+import collections
 from importlib import import_module
 from pprint import pformat
 
-from . import default_settings
+from scrapy.settings import default_settings
+
+
+if six.PY2:
+    MutableMapping = collections.MutableMapping
+else:
+    MutableMapping = collections.abc.MutableMapping
 
 
 SETTINGS_PRIORITIES = {

scrapy/settings/default_settings.py

@@ -87,6 +87,7 @@ DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClien
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
 DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
                                      # also allowing negotiation
+DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
 
 DOWNLOADER_MIDDLEWARES = {}
 
@@ -244,6 +245,7 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False
+ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser'
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'

scrapy/utils/conf.py

@@ -4,7 +4,10 @@ import numbers
 from operator import itemgetter
 
 import six
-from six.moves.configparser import SafeConfigParser
+if six.PY2:
+    from ConfigParser import SafeConfigParser as ConfigParser
+else:
+    from configparser import ConfigParser
 
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
@@ -92,9 +95,9 @@ def init_env(project='default', set_syspath=True):
 
 
 def get_config(use_closest=True):
-    """Get Scrapy config file as a SafeConfigParser"""
+    """Get Scrapy config file as a ConfigParser"""
     sources = get_sources(use_closest)
-    cfg = SafeConfigParser()
+    cfg = ConfigParser()
     cfg.read(sources)
     return cfg
 

scrapy/utils/datatypes.py

@@ -6,13 +6,20 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import copy
-import six
+import collections
 import warnings
-from collections import OrderedDict, Mapping
+
+import six
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
+if six.PY2:
+    Mapping = collections.Mapping
+else:
+    Mapping = collections.abc.Mapping
+
+
 class MultiValueDictKeyError(KeyError):
     def __init__(self, *args, **kwargs):
         warnings.warn(
@@ -245,6 +252,13 @@ class MergeDict(object):
     first occurrence will be used.
     """
     def __init__(self, *dicts):
+        if not six.PY2:
+            warnings.warn(
+                "scrapy.utils.datatypes.MergeDict is deprecated in favor "
+                "of collections.ChainMap (introduced in Python 3.3)",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
         self.dicts = dicts
 
     def __getitem__(self, key):
@@ -289,7 +303,7 @@ class MergeDict(object):
         return self.__copy__()
 
 
-class LocalCache(OrderedDict):
+class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 
     Older items expires first.

scrapy/utils/ssl.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+import OpenSSL._util as pyOpenSSLutil
+
+from scrapy.utils.python import to_native_str
+
+
+def ffi_buf_to_string(buf):
+    return to_native_str(pyOpenSSLutil.ffi.string(buf))
+
+
+def x509name_to_string(x509name):
+    # from OpenSSL.crypto.X509Name.__repr__
+    result_buffer = pyOpenSSLutil.ffi.new("char[]", 512)
+    pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer))
+
+    return ffi_buf_to_string(result_buffer)
+
+
+def get_temp_key_info(ssl_object):
+    if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'):  # requires OpenSSL 1.0.2
+        return None
+
+    # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
+    temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
+    if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
+        return None
+    temp_key = temp_key_p[0]
+    if temp_key == pyOpenSSLutil.ffi.NULL:
+        return None
+    temp_key = pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)
+    key_info = []
+    key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)
+    if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:
+        key_info.append('RSA')
+    elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH:
+        key_info.append('DH')
+    elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:
+        key_info.append('ECDH')
+        ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)
+        ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
+        nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key))
+        cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
+        if cname == pyOpenSSLutil.ffi.NULL:
+            cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
+        key_info.append(ffi_buf_to_string(cname))
+    else:
+        key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
+    key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key))
+    return ', '.join(key_info)

scrapy/utils/template.py

@@ -18,7 +18,7 @@ def render_templatefile(path, **kwargs):
         os.remove(path)
 
 
-CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]')
+CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
 def string_camelcase(string):
     """ Convert a word  to its CamelCase version and remove invalid chars
 

setup.py

@@ -53,7 +53,6 @@ setup(
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
@@ -63,19 +62,21 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
     install_requires=[
-        'Twisted>=13.1.0;python_version!="3.4"',
-        'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',
-        'w3lib>=1.17.0',
-        'queuelib',
-        'lxml',
-        'pyOpenSSL',
-        'cssselect>=0.9',
-        'six>=1.5.2',
-        'parsel>=1.5',
+        'Twisted>=16.0.0;python_version=="2.7"',
+        'Twisted>=17.9.0;python_version>="3.5"',
+        'cryptography>=2.0',
+        'cssselect>=0.9.1',
+        'lxml>=3.5.0',
+        'parsel>=1.5.0',
         'PyDispatcher>=2.0.5',
-        'service_identity',
+        'pyOpenSSL>=16.2.0',
+        'queuelib>=1.4.2',
+        'service_identity>=16.0.0',
+        'six>=1.10.0',
+        'w3lib>=1.17.0',
+        'zope.interface>=4.1.3',
     ],
     extras_require=extras_require,
 )

tests/__init__.py

@@ -35,3 +35,18 @@ def get_testdata(*paths):
     path = os.path.join(tests_datadir, *paths)
     with open(path, 'rb') as f:
         return f.read()
+
+
+# FIXME: delete after dropping py2 support
+# Monkey patch the unittest module to prevent the
+# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
+import six
+if six.PY2:
+    import unittest
+    import twisted.trial.unittest
+    if not getattr(unittest.TestCase, 'assertRegex', None):
+        unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
+    if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
+        unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
+    if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
+        twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp

tests/constraints.txt

@@ -1,2 +1 @@
-Twisted!=18.4.0
-lxml!=4.2.2
+Twisted!=18.4.0

tests/requirements-py2.txt

@@ -1,14 +1,15 @@
 # Tests requirements
-mock
+brotlipy
+jmespath
 mitmproxy==0.10.1
+mock
 netlib==0.10.1
 pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
-jmespath
-brotlipy
 testfixtures
+
 # optional for shell wrapper tests
 bpython
 ipython<6.0

tests/requirements-py3.txt

@@ -1,13 +1,14 @@
+# Tests requirements
+jmespath
+leveldb; sys_platform != "win32"
 pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
 testfixtures
-jmespath
-leveldb; sys_platform != "win32"
-botocore
+
 # optional for shell wrapper tests
 bpython
-ipython
 brotlipy
+ipython
 pywin32; sys_platform == "win32"

tests/test_command_parse.py

@@ -170,7 +170,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find callback""", _textmode(stderr))
 
     @defer.inlineCallbacks
@@ -195,7 +195,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
 
     @defer.inlineCallbacks
@@ -203,7 +203,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
 
     @defer.inlineCallbacks
     def test_crawlspider_no_matching_rule(self):
@@ -211,5 +211,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         status, out, stderr = yield self.execute(
             ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
         )
-        self.assertRegexpMatches(_textmode(out), """# Scraped Items  -+\n\[\]""")
+        self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))

tests/test_downloader_handlers.py

@@ -8,6 +8,7 @@ try:
 except ImportError:
     import mock
 
+from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.protocols.policies import WrappingFactory
 from twisted.python.filepath import FilePath
@@ -498,6 +499,24 @@ class Http11TestCase(HttpTestCase):
 class Https11TestCase(Http11TestCase):
     scheme = 'https'
 
+    tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"'
+
+    @defer.inlineCallbacks
+    def test_tls_logging(self):
+        download_handler = self.download_handler_cls(Settings({
+            'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True,
+        }))
+        try:
+            with LogCapture() as log_capture:
+                request = Request(self.getURL('file'))
+                d = download_handler.download_request(request, Spider('foo'))
+                d.addCallback(lambda r: r.body)
+                d.addCallback(self.assertEqual, b"0123456789")
+                yield d
+                log_capture.check_present(('scrapy.core.downloader.tls', 'DEBUG', self.tls_log_message))
+        finally:
+            yield download_handler.close()
+
 
 class Https11WrongHostnameTestCase(Http11TestCase):
     scheme = 'https'
@@ -518,6 +537,7 @@ class Https11InvalidDNSId(Https11TestCase):
         super(Https11InvalidDNSId, self).setUp()
         self.host = '127.0.0.1'
 
+
 class Https11InvalidDNSPattern(Https11TestCase):
     """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
 
@@ -529,6 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
             from service_identity.exceptions import CertificateError
         except ImportError:
             raise unittest.SkipTest("cryptography lib is too old")
+        self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
         super(Https11InvalidDNSPattern, self).setUp()
 
 

tests/test_downloadermiddleware_cookies.py

@@ -13,7 +13,7 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 class CookiesMiddlewareTest(TestCase):
 
     def assertCookieValEqual(self, first, second, msg=None):
-        cookievaleq = lambda cv: re.split(';\s*', cv.decode('latin1'))
+        cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1'))
         return self.assertEqual(
             sorted(cookievaleq(first)),
             sorted(cookievaleq(second)), msg)

tests/test_downloadermiddleware_robotstxt.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
-import re
 from twisted.internet import reactor, error
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
 from twisted.python import failure
@@ -11,6 +10,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.settings import Settings
 from tests import mock
+from tests.test_robotstxt_interface import rerp_available, reppy_available
 
 
 class RobotsTxtMiddlewareTest(unittest.TestCase):
@@ -31,19 +31,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def _get_successful_crawler(self):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
-        ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
-        User-Agent: *
-        Disallow: /admin/
-        Disallow: /static/
-
-        # taken from https://en.wikipedia.org/robots.txt
-        Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
-        Disallow: /wiki/Käyttäjä:
-
-        User-Agent: UnicödeBöt
-        Disallow: /some/randome/page.html
-        '''.encode('utf-8'))
+        ROBOTS = u"""
+User-Agent: *
+Disallow: /admin/
+Disallow: /static/
+# taken from https://en.wikipedia.org/robots.txt
+Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+Disallow: /wiki/Käyttäjä:
+User-Agent: UnicödeBöt
+Disallow: /some/randome/page.html
+""".encode('utf-8')
         response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
+
         def return_response(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
@@ -80,6 +79,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
         response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2')
+
         def return_response(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
@@ -102,6 +102,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         crawler = self.crawler
         crawler.settings.set('ROBOTSTXT_OBEY', True)
         response = Response('http://site.local/robots.txt')
+
         def return_response(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.callback, response)
@@ -121,6 +122,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def test_robotstxt_error(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         err = error.DNSLookupError('Robotstxt address not found')
+
         def return_failure(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(err))
@@ -136,6 +138,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
     def test_robotstxt_immediate_error(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
         err = error.DNSLookupError('Robotstxt address not found')
+
         def immediate_failure(request, spider):
             deferred = Deferred()
             deferred.errback(failure.Failure(err))
@@ -147,6 +150,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
 
     def test_ignore_robotstxt_request(self):
         self.crawler.settings.set('ROBOTSTXT_OBEY', True)
+
         def ignore_request(request, spider):
             deferred = Deferred()
             reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest()))
@@ -170,3 +174,21 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
         spider = None  # not actually used
         return self.assertFailure(maybeDeferred(middleware.process_request, request, spider),
                                   IgnoreRequest)
+
+
+class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
+    if not rerp_available():
+        skip = "Rerp parser is not installed"
+
+    def setUp(self):
+        super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
+        self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
+
+
+class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
+    if not reppy_available():
+        skip = "Reppy parser is not installed"
+
+    def setUp(self):
+        super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
+        self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')

tests/test_engine.py

@@ -40,9 +40,9 @@ class TestSpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
 
-    itemurl_re = re.compile("item\d+.html")
-    name_re = re.compile("<h1>(.*?)</h1>", re.M)
-    price_re = re.compile(">Price: \$(.*?)<", re.M)
+    itemurl_re = re.compile(r"item\d+.html")
+    name_re = re.compile(r"<h1>(.*?)</h1>", re.M)
+    price_re = re.compile(r">Price: \$(.*?)<", re.M)
 
     item_cls = TestItem
 

tests/test_exporters.py

@@ -103,7 +103,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
         return PythonItemExporter(binary=False, **kwargs)
 
     def test_invalid_option(self):
-        with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"):
+        with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):
             PythonItemExporter(invalid_option='something')
 
     def test_nested_item(self):

tests/test_http_headers.py

@@ -147,11 +147,11 @@ class HeadersTest(unittest.TestCase):
         self.assertEqual(h1.getlist('hey'), [b'5'])
 
     def test_invalid_value(self):
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers, {'foo': object()})
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers().__setitem__, 'foo', object())
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers().setdefault, 'foo', object())
-        self.assertRaisesRegexp(TypeError, 'Unsupported value type',
-                                Headers().setlist, 'foo', [object()])
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers, {'foo': object()})
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers().__setitem__, 'foo', object())
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers().setdefault, 'foo', object())
+        self.assertRaisesRegex(TypeError, 'Unsupported value type',
+                               Headers().setlist, 'foo', [object()])

tests/test_http_request.py

@@ -989,9 +989,9 @@ class FormRequestTest(RequestTest):
 
         xpath = u"//form[@name='\u03b1']"
         encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
-        self.assertRaisesRegexp(ValueError, re.escape(encoded),
-                                self.request_class.from_response,
-                                response, formxpath=xpath)
+        self.assertRaisesRegex(ValueError, re.escape(encoded),
+                               self.request_class.from_response,
+                               response, formxpath=xpath)
 
     def test_from_response_button_submit(self):
         response = _buildresponse(

tests/test_http_response.py

@@ -135,9 +135,9 @@ class BaseResponseTest(unittest.TestCase):
         r = self.response_class("http://example.com", body=b'hello')
         if self.response_class == Response:
             msg = "Response content isn't text"
-            self.assertRaisesRegexp(AttributeError, msg, getattr, r, 'text')
-            self.assertRaisesRegexp(NotSupported, msg, r.css, 'body')
-            self.assertRaisesRegexp(NotSupported, msg, r.xpath, '//body')
+            self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text')
+            self.assertRaisesRegex(NotSupported, msg, r.css, 'body')
+            self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body')
         else:
             r.text
             r.css('body')
@@ -425,13 +425,13 @@ class TextResponseTest(BaseResponseTest):
 
     def test_follow_selector_list(self):
         resp = self._links_response()
-        self.assertRaisesRegexp(ValueError, 'SelectorList',
-                                resp.follow, resp.css('a'))
+        self.assertRaisesRegex(ValueError, 'SelectorList',
+                               resp.follow, resp.css('a'))
 
     def test_follow_selector_invalid(self):
         resp = self._links_response()
-        self.assertRaisesRegexp(ValueError, 'Unsupported',
-                                resp.follow, resp.xpath('count(//div)')[0])
+        self.assertRaisesRegex(ValueError, 'Unsupported',
+                               resp.follow, resp.xpath('count(//div)')[0])
 
     def test_follow_selector_attribute(self):
         resp = self._links_response()
@@ -443,8 +443,8 @@ class TextResponseTest(BaseResponseTest):
             url='http://example.com',
             body=b'<html><body><a name=123>click me</a></body></html>',
         )
-        self.assertRaisesRegexp(ValueError, 'no href',
-                                resp.follow, resp.css('a')[0])
+        self.assertRaisesRegex(ValueError, 'no href',
+                               resp.follow, resp.css('a')[0])
 
     def test_follow_whitespace_selector(self):
         resp = self.response_class(

tests/test_linkextractors.py

@@ -288,7 +288,7 @@ class Base:
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
 
             def process_value(value):
-                m = re.search("javascript:goToPage\('(.*?)'", value)
+                m = re.search(r"javascript:goToPage\('(.*?)'", value)
                 if m:
                     return m.group(1)
 

tests/test_loader.py

@@ -691,7 +691,7 @@ class SelectortemLoaderTest(unittest.TestCase):
         self.assertTrue(l.selector)
         l.add_css('url', 'a::attr(href)')
         self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
-        l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)')
+        l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
         self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])
 
 

tests/test_robotstxt_interface.py

@@ -0,0 +1,142 @@
+# coding=utf-8
+from twisted.trial import unittest
+from scrapy.utils.python import to_native_str
+
+
+def reppy_available():
+    # check if reppy parser is installed
+    try:
+        from reppy.robots import Robots
+    except ImportError:
+        return False
+    return True
+
+
+def rerp_available():
+    # check if robotexclusionrulesparser is installed
+    try:
+        from robotexclusionrulesparser import RobotExclusionRulesParser
+    except ImportError:
+        return False
+    return True
+
+
+class BaseRobotParserTest:
+    def _setUp(self, parser_cls):
+        self.parser_cls = parser_cls
+
+    def test_allowed(self):
+        robotstxt_robotstxt_body = ("User-agent: * \n"
+                                    "Disallow: /disallowed \n"
+                                    "Allow: /allowed \n"
+                                    "Crawl-delay: 10".encode('utf-8'))
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://www.site.local/allowed", "*"))
+        self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))
+
+    def test_allowed_wildcards(self):
+        robotstxt_robotstxt_body = """User-agent: first
+                                Disallow: /disallowed/*/end$    
+
+                                User-agent: second
+                                Allow: /*allowed
+                                Disallow: /
+                                """.encode('utf-8')
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+
+        self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first"))
+        self.assertFalse(rp.allowed("https://www.site.local/disallowed/xyz/end", "first"))
+        self.assertFalse(rp.allowed("https://www.site.local/disallowed/abc/end", "first"))
+        self.assertTrue(rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first"))
+
+        self.assertTrue(rp.allowed("https://www.site.local/allowed", "second"))
+        self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second"))
+        self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
+
+    def test_length_based_precedence(self):
+        robotstxt_robotstxt_body = ("User-agent: * \n"
+                                    "Disallow: / \n"
+                                    "Allow: /page".encode('utf-8'))
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
+
+    def test_order_based_precedence(self):
+        robotstxt_robotstxt_body = ("User-agent: * \n"
+                                    "Disallow: / \n"
+                                    "Allow: /page".encode('utf-8'))
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertFalse(rp.allowed("https://www.site.local/page", "*"))
+
+    def test_empty_response(self):
+        """empty response should equal 'allow all'"""
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b'')
+        self.assertTrue(rp.allowed("https://site.local/", "*"))
+        self.assertTrue(rp.allowed("https://site.local/", "chrome"))
+        self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
+        self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
+
+    def test_garbage_response(self):
+        """garbage response should be discarded, equal 'allow all'"""
+        robotstxt_robotstxt_body = b'GIF89a\xd3\x00\xfe\x00\xa2'
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://site.local/", "*"))
+        self.assertTrue(rp.allowed("https://site.local/", "chrome"))
+        self.assertTrue(rp.allowed("https://site.local/index.html", "*"))
+        self.assertTrue(rp.allowed("https://site.local/disallowed", "*"))
+
+    def test_unicode_url_and_useragent(self):
+        robotstxt_robotstxt_body = u"""
+        User-Agent: *
+        Disallow: /admin/
+        Disallow: /static/
+        # taken from https://en.wikipedia.org/robots.txt
+        Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
+        Disallow: /wiki/Käyttäjä:
+
+        User-Agent: UnicödeBöt
+        Disallow: /some/randome/page.html""".encode('utf-8')
+        rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body)
+        self.assertTrue(rp.allowed("https://site.local/", "*"))
+        self.assertFalse(rp.allowed("https://site.local/admin/", "*"))
+        self.assertFalse(rp.allowed("https://site.local/static/", "*"))
+        self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt"))
+        self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*"))
+        self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*"))
+        self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*"))
+        self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt"))
+
+
+class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    def setUp(self):
+        from scrapy.robotstxt import PythonRobotParser
+        super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
+
+    def test_length_based_precedence(self):
+        raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
+
+    def test_allowed_wildcards(self):
+        raise unittest.SkipTest("RobotFileParser does not support wildcards.")
+
+
+class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    if not reppy_available():
+        skip = "Reppy parser is not installed"
+
+    def setUp(self):
+        from scrapy.robotstxt import ReppyRobotParser
+        super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
+
+    def test_order_based_precedence(self):
+        raise unittest.SkipTest("Rerp does not support order based directives precedence.")
+
+
+class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
+    if not rerp_available():
+        skip = "Rerp parser is not installed"
+
+    def setUp(self):
+        from scrapy.robotstxt import RerpRobotParser
+        super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
+
+    def test_length_based_precedence(self):
+        raise unittest.SkipTest("Rerp does not support length based directives precedence.")

tests/test_selector.py

@@ -85,5 +85,5 @@ class SelectorTestCase(unittest.TestCase):
             x.__class__.__name__
 
     def test_selector_bad_args(self):
-        with self.assertRaisesRegexp(ValueError, 'received both response and text'):
+        with self.assertRaisesRegex(ValueError, 'received both response and text'):
             Selector(TextResponse(url='http://example.com', body=b''), text=u'')

tests/test_spider.py

@@ -595,5 +595,5 @@ class NoParseMethodSpiderTest(unittest.TestCase):
         resp = TextResponse(url="http://www.example.com/random_url", body=text)
 
         exc_msg = 'Spider.parse callback is not defined'
-        with self.assertRaisesRegexp(NotImplementedError, exc_msg):
+        with self.assertRaisesRegex(NotImplementedError, exc_msg):
             spider.parse(resp)

tests/test_spiderloader/__init__.py

@@ -84,8 +84,8 @@ class SpiderLoaderTest(unittest.TestCase):
         module = 'tests.test_spiderloader.test_spiders.spider1'
         runner = CrawlerRunner({'SPIDER_MODULES': [module]})
 
-        self.assertRaisesRegexp(KeyError, 'Spider not found',
-                                runner.create_crawler, 'spider2')
+        self.assertRaisesRegex(KeyError, 'Spider not found',
+                               runner.create_crawler, 'spider2')
 
         crawler = runner.create_crawler('spider1')
         self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))

tests/test_utils_datatypes.py

@@ -1,11 +1,18 @@
 import copy
 import unittest
-from collections import Mapping, MutableMapping
+
+import six
+if six.PY2:
+    from collections import Mapping, MutableMapping
+else:
+    from collections.abc import Mapping, MutableMapping
 
 from scrapy.utils.datatypes import CaselessDict, SequenceExclude
 
+
 __doctests__ = ['scrapy.utils.datatypes']
 
+
 class CaselessDictTest(unittest.TestCase):
 
     def test_init_dict(self):

tests/test_utils_url.py

@@ -233,8 +233,8 @@ for k, args in enumerate ([
     setattr (GuessSchemeTest, t_method.__name__, t_method)
 
 # TODO: the following tests do not pass with current implementation
-for k, args in enumerate ([
-            ('C:\absolute\path\to\a\file.html',     'file://',
+for k, args in enumerate([
+            (r'C:\absolute\path\to\a\file.html', 'file://',
              'Windows filepath are not supported for scrapy shell'),
         ], start=1):
     t_method = create_skipped_scheme_t(args)

tox.ini

@@ -11,10 +11,10 @@ deps =
     -ctests/constraints.txt
     -rrequirements-py2.txt
     # Extras
-    botocore
+    botocore>=1.3.23
     google-cloud-storage
-    Pillow != 3.0.0
     leveldb
+    Pillow>=3.4.2
     -rtests/requirements-py2.txt
 passenv =
     S3_TEST_FILE_URI
@@ -25,72 +25,74 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
-[testenv:trusty]
+[testenv:py27-pinned]
 basepython = python2.7
 deps =
-    pyOpenSSL==0.13
-    lxml==3.3.3
-    Twisted==13.2.0
-    boto==2.20.1
-    Pillow==2.3.0
+    -ctests/constraints.txt
+    cryptography==2.0
     cssselect==0.9.1
-    zope.interface==4.0.5
+    lxml==3.5.0
+    parsel==1.5.0
+    PyDispatcher==2.0.5
+    pyOpenSSL==16.2.0
+    queuelib==1.4.2
+    service_identity==16.0.0
+    six==1.10.0
+    Twisted==16.0.0
+    w3lib==1.17.0
+    zope.interface==4.1.3
     -rtests/requirements-py2.txt
-
-[testenv:jessie]
-# https://packages.debian.org/en/jessie/python/
-# https://packages.debian.org/en/jessie/zope/
-basepython = python2.7
-deps =
-    cryptography==0.6.1
-    pyOpenSSL==0.14
-    lxml==3.4.0
-    Twisted==14.0.2
-    boto==2.34.0
-    Pillow==2.6.1
-    cssselect==0.9.1
-    zope.interface==4.1.1
-    -rtests/requirements-py2.txt
-# Not used directly but allows boto GCE plugins to load.
-# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
-    google-compute-engine==2.8.12
-
-[testenv:trunk]
-basepython = python2.7
-commands =
-    pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib
-    pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
-    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
+    # Extras
+    botocore==1.3.23
+    Pillow==3.4.2
 
 [testenv:pypy]
 basepython = pypy
 commands =
     py.test {posargs:scrapy tests}
 
-[testenv:py34]
-basepython = python3.4
+[testenv:py35]
+basepython = python3.5
 deps =
     -ctests/constraints.txt
     -rrequirements-py3.txt
-    # Extras
-    Pillow
     -rtests/requirements-py3.txt
+    # Extras
+    botocore>=1.3.23
+    Pillow>=3.4.2
 
-[testenv:py35]
+[testenv:py35-pinned]
 basepython = python3.5
-deps = {[testenv:py34]deps}
+deps =
+    -ctests/constraints.txt
+    cryptography==2.0
+    cssselect==0.9.1
+    lxml==3.5.0
+    parsel==1.5.0
+    PyDispatcher==2.0.5
+    pyOpenSSL==16.2.0
+    queuelib==1.4.2
+    service_identity==16.0.0
+    six==1.10.0
+    Twisted==17.9.0
+    w3lib==1.17.0
+    zope.interface==4.1.3
+    -rtests/requirements-py3.txt
+    # Extras
+    botocore==1.3.23
+    Pillow==3.4.2
 
 [testenv:py36]
 basepython = python3.6
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
 
 [testenv:py37]
 basepython = python3.7
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
 
 [testenv:pypy3]
 basepython = pypy3
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
 commands =
     py.test {posargs:scrapy tests}
 
@@ -116,3 +118,17 @@ changedir = {[docs]changedir}
 deps = {[docs]deps}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+[testenv:py37-extra-deps]
+basepython = python3.7
+deps =
+    {[testenv:py35]deps}
+    reppy
+    robotexclusionrulesparser
+
+[testenv:py27-extra-deps]
+basepython = python2.7
+deps =
+    {[testenv]deps}
+    reppy
+    robotexclusionrulesparser