diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8cecb7ad4..70affe63f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.6.0 +current_version = 1.7.0 commit = True tag = True tag_name = {new_version} diff --git a/.travis.yml b/.travis.yml index 08b0bf119..0190a7f4d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +dist: xenial branches: only: - master @@ -6,26 +7,28 @@ branches: - /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/ matrix: include: - - python: 2.7 - env: TOXENV=py27 - - python: 2.7 - env: TOXENV=jessie - - python: 2.7 - env: TOXENV=pypy - - python: 2.7 - env: TOXENV=pypy3 - - python: 3.4 - env: TOXENV=py34 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - dist: xenial - sudo: true - - python: 3.6 - env: TOXENV=docs + - env: TOXENV=py27 + python: 2.7 + - env: TOXENV=py27-pinned + python: 2.7 + - env: TOXENV=py27-extra-deps + python: 2.7 + - env: TOXENV=pypy + python: 2.7 + - env: TOXENV=pypy3 + python: 3.5 + - env: TOXENV=py35 + python: 3.5 + - env: TOXENV=py35-pinned + python: 3.5 + - env: TOXENV=py36 + python: 3.6 + - env: TOXENV=py37 + python: 3.7 + - env: TOXENV=py37-extra-deps + python: 3.7 + - env: TOXENV=docs + python: 3.6 install: - | if [ "$TOXENV" = "pypy" ]; then diff --git a/README.rst b/README.rst index c28d217ff..bd82bff06 100644 --- a/README.rst +++ b/README.rst @@ -40,7 +40,7 @@ https://scrapy.org Requirements ============ -* Python 2.7 or Python 3.4+ +* Python 2.7 or Python 3.5+ * Works on Linux, Windows, Mac OSX, BSD Install diff --git a/docs/conf.py b/docs/conf.py index 80b8e3e90..f49f79cd5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,6 +30,7 @@ extensions = [ 'scrapydocs', 'sphinx.ext.autodoc', 'sphinx.ext.coverage', + 'sphinx.ext.intersphinx', ] # Add any paths that contain templates here, relative to this directory. @@ -74,6 +75,8 @@ language = 'en' # List of documents that shouldn't be included in the build. #unused_docs = [] +exclude_patterns = ['build'] + # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ['.build'] @@ -250,3 +253,11 @@ coverage_ignore_pyobjects = [ # Private exception used by the command-line interface implementation. r'^scrapy\.exceptions\.UsageError', ] + + +# Options for the InterSphinx extension +# ------------------------------------- + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), +} diff --git a/docs/contributing.rst b/docs/contributing.rst index b4f91ea8d..28dea74de 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -171,6 +171,8 @@ Scrapy: See https://help.github.com/articles/setting-your-username-in-git/ for setup instructions. +.. _documentation-policies: + Documentation policies ====================== @@ -196,6 +198,8 @@ Tests Tests are implemented using the `Twisted unit-testing framework`_, running tests requires `tox`_. +.. _running-tests: + Running tests ------------- diff --git a/docs/faq.rst b/docs/faq.rst index 7105baeef..9733471bf 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars What Python versions does Scrapy support? ----------------------------------------- -Scrapy is supported under Python 2.7 and Python 3.4+ +Scrapy is supported under Python 2.7 and Python 3.5+ under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). Python 2.6 support was dropped starting at Scrapy 0.20. Python 3 support was added in Scrapy 1.1. @@ -329,6 +329,8 @@ I'm scraping a XML document and my XPath selector doesn't return any items You may need to remove namespaces. See :ref:`removing-namespaces`. +.. _faq-split-item: + How to split an item into multiple items in an item pipeline? ------------------------------------------------------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index daec7fcb7..2bf98dbdc 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -7,7 +7,7 @@ Installation guide Installing Scrapy ================= -Scrapy runs on Python 2.7 and Python 3.4 or above +Scrapy runs on Python 2.7 and Python 3.5 or above under CPython (default Python implementation) and PyPy (starting with PyPy 5.9). If you're using `Anaconda`_ or `Miniconda`_, you can install the package from diff --git a/docs/news.rst b/docs/news.rst index 75e3eef91..ce5b8b406 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,346 @@ Release notes ============= +.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0, + planned for Q4 2019 or Q1 2020, will support **Python 3 only**. + +Scrapy 1.7.3 (2019-08-01) +------------------------- + +Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`). + +Scrapy 1.7.2 (2019-07-23) +------------------------- + +Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`). + + +Scrapy 1.7.1 (2019-07-18) +------------------------- + +Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI. + +.. _release-1.7.0: + +Scrapy 1.7.0 (2019-07-18) +------------------------- + +.. note:: Make sure you install Scrapy 1.7.1. The Scrapy 1.7.0 package in PyPI + is the result of an erroneous commit tagging and does not include all + the changes described below. + +Highlights: + +* Improvements for crawls targeting multiple domains +* A cleaner way to pass arguments to callbacks +* A new class for JSON requests +* Improvements for rule-based spiders +* New features for feed exports + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* ``429`` is now part of the :setting:`RETRY_HTTP_CODES` setting by default + + This change is **backward incompatible**. If you don’t want to retry + ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly. + +* :class:`~scrapy.crawler.Crawler`, + :class:`CrawlerRunner.crawl ` and + :class:`CrawlerRunner.create_crawler ` + no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they + only accept a :class:`~scrapy.spiders.Spider` subclass now. + + :class:`~scrapy.spiders.Spider` subclass instances were never meant to + work, and they were not working as one would expect: instead of using the + passed :class:`~scrapy.spiders.Spider` subclass instance, their + :class:`~scrapy.spiders.Spider.from_crawler` method was called to generate + a new instance. + +* Non-default values for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting + may stop working. Scheduler priority queue classes now need to handle + :class:`~scrapy.http.Request` objects instead of arbitrary Python data + structures. + +See also :ref:`1.7-deprecation-removals` below. + + +New features +~~~~~~~~~~~~ + +* A new scheduler priority queue, + :class:`scrapy.pqueues.DownloaderAwarePriorityQueue`, may be + :ref:`enabled ` for a significant + scheduling improvement on crawls targetting multiple web domains, at the + cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + +* A new :attr:`Request.cb_kwargs ` attribute + provides a cleaner way to pass keyword arguments to callback methods + (:issue:`1138`, :issue:`3563`) + +* A new :class:`~scrapy.http.JSONRequest` class offers a more convenient way + to build JSON requests (:issue:`3504`, :issue:`3505`) + +* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule` + constructor now receives the :class:`~scrapy.http.Response` object that + originated the request as its second argument (:issue:`3682`) + +* A new ``restrict_text`` parameter for the + :attr:`LinkExtractor ` + constructor allows filtering links by linking text (:issue:`3622`, + :issue:`3635`) + +* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL + for feeds exported to Amazon S3 (:issue:`3607`) + +* A new :setting:`FEED_STORAGE_FTP_ACTIVE` setting allows using FTP’s active + connection mode for feeds exported to FTP servers (:issue:`3829`) + +* A new :setting:`METAREFRESH_IGNORE_TAGS` setting allows overriding which + HTML tags are ignored when searching a response for HTML meta tags that + trigger a redirect (:issue:`1422`, :issue:`3768`) + +* A new :reqmeta:`redirect_reasons` request meta key exposes the reason + (status code, meta refresh) behind every followed redirect (:issue:`3581`, + :issue:`3687`) + +* The ``SCRAPY_CHECK`` variable is now set to the ``true`` string during runs + of the :command:`check` command, which allows :ref:`detecting contract + check runs from code ` (:issue:`3704`, + :issue:`3739`) + +* A new :meth:`Item.deepcopy() ` method makes it + easier to :ref:`deep-copy items ` (:issue:`1493`, + :issue:`3671`) + +* :class:`~scrapy.extensions.corestats.CoreStats` also logs + ``elapsed_time_seconds`` now (:issue:`3638`) + +* Exceptions from :class:`~scrapy.loader.ItemLoader` :ref:`input and output + processors ` are now more verbose + (:issue:`3836`, :issue:`3840`) + +* :class:`~scrapy.crawler.Crawler`, + :class:`CrawlerRunner.crawl ` and + :class:`CrawlerRunner.create_crawler ` + now fail gracefully if they receive a :class:`~scrapy.spiders.Spider` + subclass instance instead of the subclass itself (:issue:`2283`, + :issue:`3610`, :issue:`3872`) + + +Bug fixes +~~~~~~~~~ + +* :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception` + is now also invoked for generators (:issue:`220`, :issue:`2061`) + +* System exceptions like KeyboardInterrupt_ are no longer caught + (:issue:`3726`) + +* :meth:`ItemLoader.load_item() ` no + longer makes later calls to :meth:`ItemLoader.get_output_value() + ` or + :meth:`ItemLoader.load_item() ` return + empty data (:issue:`3804`, :issue:`3819`) + +* The images pipeline (:class:`~scrapy.pipelines.images.ImagesPipeline`) no + longer ignores these Amazon S3 settings: :setting:`AWS_ENDPOINT_URL`, + :setting:`AWS_REGION_NAME`, :setting:`AWS_USE_SSL`, :setting:`AWS_VERIFY` + (:issue:`3625`) + +* Fixed a memory leak in :class:`~scrapy.pipelines.media.MediaPipeline` + affecting, for example, non-200 responses and exceptions from custom + middlewares (:issue:`3813`) + +* Requests with private callbacks are now correctly unserialized from disk + (:issue:`3790`) + +* :meth:`FormRequest.from_response() ` + now handles invalid methods like major web browsers (:issue:`3777`, + :issue:`3794`) + + +Documentation +~~~~~~~~~~~~~ + +* A new topic, :ref:`topics-dynamic-content`, covers recommended approaches + to read dynamically-loaded data (:issue:`3703`) + +* :ref:`topics-broad-crawls` now features information about memory usage + (:issue:`1264`, :issue:`3866`) + +* The documentation of :class:`~scrapy.spiders.Rule` now covers how to access + the text of a link when using :class:`~scrapy.spiders.CrawlSpider` + (:issue:`3711`, :issue:`3712`) + +* A new section, :ref:`httpcache-storage-custom`, covers writing a custom + cache storage backend for + :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` + (:issue:`3683`, :issue:`3692`) + +* A new :ref:`FAQ ` entry, :ref:`faq-split-item`, explains what to do + when you want to split an item into multiple items from an item pipeline + (:issue:`2240`, :issue:`3672`) + +* Updated the :ref:`FAQ entry about crawl order ` to explain why + the first few requests rarely follow the desired order (:issue:`1739`, + :issue:`3621`) + +* The :setting:`LOGSTATS_INTERVAL` setting (:issue:`3730`), the + :meth:`FilesPipeline.file_path ` + and + :meth:`ImagesPipeline.file_path ` + methods (:issue:`2253`, :issue:`3609`) and the + :meth:`Crawler.stop() ` method (:issue:`3842`) + are now documented + +* Some parts of the documentation that were confusing or misleading are now + clearer (:issue:`1347`, :issue:`1789`, :issue:`2289`, :issue:`3069`, + :issue:`3615`, :issue:`3626`, :issue:`3668`, :issue:`3670`, :issue:`3673`, + :issue:`3728`, :issue:`3762`, :issue:`3861`, :issue:`3882`) + +* Minor documentation fixes (:issue:`3648`, :issue:`3649`, :issue:`3662`, + :issue:`3674`, :issue:`3676`, :issue:`3694`, :issue:`3724`, :issue:`3764`, + :issue:`3767`, :issue:`3791`, :issue:`3797`, :issue:`3806`, :issue:`3812`) + +.. _1.7-deprecation-removals: + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +The following deprecated APIs have been removed (:issue:`3578`): + +* ``scrapy.conf`` (use :attr:`Crawler.settings + `) + +* From ``scrapy.core.downloader.handlers``: + + * ``http.HttpDownloadHandler`` (use ``http10.HTTP10DownloadHandler``) + +* ``scrapy.loader.ItemLoader._get_values`` (use ``_get_xpathvalues``) + +* ``scrapy.loader.XPathItemLoader`` (use :class:`~scrapy.loader.ItemLoader`) + +* ``scrapy.log`` (see :ref:`topics-logging`) + +* From ``scrapy.pipelines``: + + * ``files.FilesPipeline.file_key`` (use ``file_path``) + + * ``images.ImagesPipeline.file_key`` (use ``file_path``) + + * ``images.ImagesPipeline.image_key`` (use ``file_path``) + + * ``images.ImagesPipeline.thumb_key`` (use ``thumb_path``) + +* From both ``scrapy.selector`` and ``scrapy.selector.lxmlsel``: + + * ``HtmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + + * ``XmlXPathSelector`` (use :class:`~scrapy.selector.Selector`) + + * ``XPathSelector`` (use :class:`~scrapy.selector.Selector`) + + * ``XPathSelectorList`` (use :class:`~scrapy.selector.Selector`) + +* From ``scrapy.selector.csstranslator``: + + * ``ScrapyGenericTranslator`` (use parsel.csstranslator.GenericTranslator_) + + * ``ScrapyHTMLTranslator`` (use parsel.csstranslator.HTMLTranslator_) + + * ``ScrapyXPathExpr`` (use parsel.csstranslator.XPathExpr_) + +* From :class:`~scrapy.selector.Selector`: + + * ``_root`` (both the constructor argument and the object property, use + ``root``) + + * ``extract_unquoted`` (use ``getall``) + + * ``select`` (use ``xpath``) + +* From :class:`~scrapy.selector.SelectorList`: + + * ``extract_unquoted`` (use ``getall``) + + * ``select`` (use ``xpath``) + + * ``x`` (use ``xpath``) + +* ``scrapy.spiders.BaseSpider`` (use :class:`~scrapy.spiders.Spider`) + +* From :class:`~scrapy.spiders.Spider` (and subclasses): + + * ``DOWNLOAD_DELAY`` (use :ref:`download_delay + `) + + * ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`) + +* ``scrapy.spiders.spiders`` (use :class:`~scrapy.spiderloader.SpiderLoader`) + +* ``scrapy.telnet`` (use :mod:`scrapy.extensions.telnet`) + +* From ``scrapy.utils.python``: + + * ``str_to_unicode`` (use ``to_unicode``) + + * ``unicode_to_str`` (use ``to_bytes``) + +* ``scrapy.utils.response.body_or_str`` + +The following deprecated settings have also been removed (:issue:`3578`): + +* ``SPIDER_MANAGER_CLASS`` (use :setting:`SPIDER_LOADER_CLASS`) + + +Deprecations +~~~~~~~~~~~~ + +* The ``queuelib.PriorityQueue`` value for the + :setting:`SCHEDULER_PRIORITY_QUEUE` setting is deprecated. Use + :class:`scrapy.pqueues.ScrapyPriorityQueue` instead. + +* ``process_request`` callbacks passed to :class:`~scrapy.spiders.Rule` that + do not accept two arguments are deprecated. + +* The following modules are deprecated: + + * ``scrapy.utils.http`` (use `w3lib.http`_) + + * ``scrapy.utils.markup`` (use `w3lib.html`_) + + * ``scrapy.utils.multipart`` (use `urllib3`_) + +* The ``scrapy.utils.datatypes.MergeDict`` class is deprecated for Python 3 + code bases. Use :class:`~collections.ChainMap` instead. (:issue:`3878`) + +* The ``scrapy.utils.gz.is_gzipped`` function is deprecated. Use + ``scrapy.utils.gz.gzip_magic_number`` instead. + +.. _urllib3: https://urllib3.readthedocs.io/en/latest/index.html +.. _w3lib.html: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.html +.. _w3lib.http: https://w3lib.readthedocs.io/en/latest/w3lib.html#module-w3lib.http + + +Other changes +~~~~~~~~~~~~~ + +* It is now possible to run all tests from the same tox_ environment in + parallel; the documentation now covers :ref:`this and other ways to run + tests ` (:issue:`3707`) + +* It is now possible to generate an API documentation coverage report + (:issue:`3806`, :issue:`3810`, :issue:`3860`) + +* The :ref:`documentation policies ` now require + docstrings_ (:issue:`3701`) that follow `PEP 257`_ (:issue:`3748`) + +* Internal fixes and cleanup (:issue:`3629`, :issue:`3643`, :issue:`3684`, + :issue:`3698`, :issue:`3734`, :issue:`3735`, :issue:`3736`, :issue:`3737`, + :issue:`3809`, :issue:`3821`, :issue:`3825`, :issue:`3827`, :issue:`3833`, + :issue:`3857`, :issue:`3877`) + .. _release-1.6.0: Scrapy 1.6.0 (2019-01-30) @@ -2471,12 +2811,22 @@ First release of Scrapy. .. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1 .. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding -.. _w3lib: https://github.com/scrapy/w3lib -.. _scrapely: https://github.com/scrapy/scrapely -.. _marshal: https://docs.python.org/2/library/marshal.html -.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py -.. _lxml: http://lxml.de/ .. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/ -.. _resource: https://docs.python.org/2/library/resource.html +.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request +.. _cssselect: https://github.com/scrapy/cssselect/ +.. _docstrings: https://docs.python.org/glossary.html#term-docstring +.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt +.. _lxml: http://lxml.de/ +.. _marshal: https://docs.python.org/2/library/marshal.html +.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator +.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator +.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr +.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _queuelib: https://github.com/scrapy/queuelib -.. _cssselect: https://github.com/SimonSapin/cssselect +.. _resource: https://docs.python.org/2/library/resource.html +.. _scrapely: https://github.com/scrapy/scrapely +.. _tox: https://pypi.python.org/pypi/tox +.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ +.. _w3lib: https://github.com/scrapy/w3lib +.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py +.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 05b1d161b..957761b76 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -126,6 +126,7 @@ response received:: if header not in response.headers: raise ContractFail('X-CustomHeader not present') +.. _detecting-contract-check-runs: Detecting check runs ==================== diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index d7add4ec4..616b56101 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -52,6 +52,8 @@ as its value. For example, if you want to disable the user-agent middleware:: Finally, keep in mind that some middlewares may need to be enabled through a particular setting. See each middleware documentation for more info. +.. _topics-downloader-middleware-custom: + Writing your own downloader middleware ====================================== @@ -961,7 +963,7 @@ precedence over the :setting:`RETRY_TIMES` setting. RETRY_HTTP_CODES ^^^^^^^^^^^^^^^^ -Default: ``[500, 502, 503, 504, 522, 524, 408]`` +Default: ``[500, 502, 503, 504, 522, 524, 408, 429]`` Which HTTP response codes to retry. Other errors (DNS lookup issues, connections lost, etc) are always retried. @@ -987,6 +989,17 @@ RobotsTxtMiddleware To make sure Scrapy respects robots.txt make sure the middleware is enabled and the :setting:`ROBOTSTXT_OBEY` setting is enabled. + This middleware has to be combined with a robots.txt_ parser. + + Scrapy ships with support for the following robots.txt_ parsers: + + * :ref:`RobotFileParser ` (default) + * :ref:`Reppy ` + * :ref:`Robotexclusionrulesparser ` + + You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` + setting. Or you can also :ref:`implement support for a new parser `. + .. reqmeta:: dont_obey_robotstxt If :attr:`Request.meta ` has @@ -994,6 +1007,74 @@ If :attr:`Request.meta ` has the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. +.. _python-robotfileparser: + +RobotFileParser +~~~~~~~~~~~~~~~ + +`RobotFileParser `_ is +Python's inbuilt ``robots.txt`` parser. The parser is fully compliant with `Martijn Koster's +1996 draft specification `_. It lacks +support for wildcard matching. Scrapy uses this parser by default. + +In order to use this parser, set: + +* :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` + +.. _rerp-parser: + +Robotexclusionrulesparser +~~~~~~~~~~~~~~~~~~~~~~~~~ + +`Robotexclusionrulesparser `_ is fully compliant +with `Martijn Koster's 1996 draft specification `_, +with support for wildcard matching. + +In order to use this parser: + +* Install `Robotexclusionrulesparser `_ by running + ``pip install robotexclusionrulesparser`` + +* Set :setting:`ROBOTSTXT_PARSER` setting to + ``scrapy.robotstxt.RerpRobotParser`` + +.. _reppy-parser: + +Reppy parser +~~~~~~~~~~~~ + +`Reppy `_ is a Python wrapper around `Robots Exclusion +Protocol Parser for C++ `_. The parser is fully compliant +with `Martijn Koster's 1996 draft specification `_, +with support for wildcard matching. Unlike +`RobotFileParser `_ and +`Robotexclusionrulesparser `_, it uses the length based +rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific +rule based on the length of the path trumps the less specific (shorter) rule. + +In order to use this parser: + +* Install `Reppy `_ by running ``pip install reppy`` + +* Set :setting:`ROBOTSTXT_PARSER` setting to + ``scrapy.robotstxt.ReppyRobotParser`` + +.. _support-for-new-robots-parser: + +Implementing support for a new parser +------------------------------------- + +You can implement support for a new robots.txt_ parser by subclassing +the abstract base class :class:`~scrapy.robotstxt.RobotParser` and +implementing the methods described below. + +.. module:: scrapy.robotstxt + :synopsis: robots.txt parser interface and implementations + +.. autoclass:: RobotParser + :members: + +.. _robots.txt: http://www.robotstxt.org/ DownloaderStats --------------- diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7e32da6f4..af541db78 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -164,9 +164,9 @@ The feeds are stored in a FTP server. * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` * Required external libraries: none -FTP supports two different connection modes: [active or passive]( -https://stackoverflow.com/a/1699163). Scrapy uses the passive connection mode -by default. To use the active connection mode instead, set the +FTP supports two different connection modes: `active or passive +`_. Scrapy uses the passive connection +mode by default. To use the active connection mode instead, set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. .. _topics-feed-storage-s3: @@ -320,8 +320,11 @@ FEED_STORAGE_FTP_ACTIVE Default: ``False`` -Whether to use [active mode](https://stackoverflow.com/a/1699163) when exporting feeds -to a FTP server. +Whether to use the active connection mode when exporting feeds to an FTP server +(``True``) or use the passive connection mode instead (``False``, default). + +For information about FTP connection modes, see `What is the difference between +active and passive FTP? `_. .. setting:: FEED_STORAGE_S3_ACL diff --git a/docs/topics/items.rst b/docs/topics/items.rst index d744fd9ea..60fbc82f8 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -157,6 +157,8 @@ To access all populated values, just use the typical `dict API`_:: [('price', 1000), ('name', 'Desktop PC')] +.. _copying-items: + Copying items ------------- diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index dea0528db..87ea43c7d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -193,6 +193,17 @@ to override some of the Scrapy settings regarding logging. Module `logging.handlers `_ Further documentation on available handlers +.. _custom-log-formats: + +Custom Log Formats +------------------ + +A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class +and making :setting:`LOG_FORMATTER` point to your new class. + +.. autoclass:: scrapy.logformatter.LogFormatter + :members: + Advanced customization ---------------------- diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 298a078a7..a6d4f0d6d 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -34,15 +34,16 @@ Here's an example showing how to run a single spider with it. # Your spider definition ... - process = CrawlerProcess({ - 'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' + process = CrawlerProcess(settings={ + 'FEED_FORMAT': 'json', + 'FEED_URI': 'items.json' }) process.crawl(MySpider) process.start() # the script will block here until the crawling is finished -Make sure to check :class:`~scrapy.crawler.CrawlerProcess` documentation to get -acquainted with its usage details. +Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess` +documentation to get acquainted with its usage details. If you are inside a Scrapy project there are some additional helpers you can use to import those components within the project. You can automatically import diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 371f21c72..12606fe47 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -30,6 +30,8 @@ Python `import search path`_. .. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path +.. _populating-settings: + Populating the settings ======================= @@ -438,9 +440,10 @@ or even enable client-side authentication (and various other things). which uses the platform's certificates to validate remote endpoints. **This is only available if you use Twisted>=14.0.** -If you do use a custom ContextFactory, make sure it accepts a ``method`` -parameter at init (this is the ``OpenSSL.SSL`` method mapping -:setting:`DOWNLOADER_CLIENT_TLS_METHOD`). +If you do use a custom ContextFactory, make sure its ``__init__`` method +accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping +:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``tls_verbose_logging`` +parameter (``bool``). .. setting:: DOWNLOADER_CLIENT_TLS_METHOD @@ -468,6 +471,20 @@ This setting must be one of these string values: We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 or above (Twisted>=14.0 if you can). +.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING + +DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING +------------------------------------- + +Default: ``False`` + +Setting this to ``True`` will enable DEBUG level messages about TLS connection +parameters after establishing HTTPS connections. The kind of information logged +depends on the versions of OpenSSL and pyOpenSSL. + +This setting is only used for the default +:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + .. setting:: DOWNLOADER_MIDDLEWARES DOWNLOADER_MIDDLEWARES @@ -538,6 +555,8 @@ amount of time between requests, but uses a random interval between 0.5 * :setti When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced per ip address instead of per domain. +.. _spider-download_delay-attribute: + You can also change this setting per spider by setting ``download_delay`` spider attribute. @@ -866,6 +885,15 @@ directives. .. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior +.. setting:: LOG_FORMATTER + +LOG_FORMATTER +------------- + +Default: :class:`scrapy.logformatter.LogFormatter` + +The class to use for :ref:`formatting log messages ` for different actions. + .. setting:: LOG_LEVEL LOG_LEVEL @@ -1113,6 +1141,16 @@ If enabled, Scrapy will respect robots.txt policies. For more information see this option is enabled by default in settings.py file generated by ``scrapy startproject`` command. +.. setting:: ROBOTSTXT_PARSER + +ROBOTSTXT_PARSER +---------------- + +Default: ``'scrapy.robotstxt.PythonRobotParser'`` + +The parser backend to use for parsing ``robots.txt`` files. For more information see +:ref:`topics-dlmw-robots`. + .. setting:: SCHEDULER SCHEDULER diff --git a/requirements-py2.txt b/requirements-py2.txt index 0771aae3a..9e6944240 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -1,10 +1,17 @@ -Twisted>=13.1.0 -lxml -pyOpenSSL -cssselect>=0.9 -queuelib -w3lib>=1.17.0 -six>=1.5.2 +parsel>=1.5.0 PyDispatcher>=2.0.5 -parsel>=1.5 -service_identity +w3lib>=1.17.0 + +pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" +queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" +cryptography>=2.0 # Earlier versions would fail to install + +# Reference versions taken from +# https://packages.ubuntu.com/xenial/python/ +# https://packages.ubuntu.com/xenial/zope/ +cssselect>=0.9.1 +lxml>=3.5.0 +service_identity>=16.0.0 +six>=1.10.0 +Twisted>=16.0.0 +zope.interface>=4.1.3 diff --git a/requirements-py3.txt b/requirements-py3.txt index 5a5d4c95a..cd183a525 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -1,10 +1,17 @@ -Twisted>=17.9.0 -lxml>=3.2.4 -pyOpenSSL>=0.13.1 -cssselect>=0.9 -queuelib>=1.1.1 -w3lib>=1.17.0 -six>=1.5.2 +parsel>=1.5.0 PyDispatcher>=2.0.5 -parsel>=1.5 -service_identity +Twisted>=17.9.0 +w3lib>=1.17.0 + +pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" +queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" +cryptography>=2.0 # Earlier versions would fail to install + +# Reference versions taken from +# https://packages.ubuntu.com/xenial/python/ +# https://packages.ubuntu.com/xenial/zope/ +cssselect>=0.9.1 +lxml>=3.5.0 +service_identity>=16.0.0 +six>=1.10.0 +zope.interface>=4.1.3 diff --git a/scrapy/VERSION b/scrapy/VERSION index dc1e644a1..bd8bf882d 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -1.6.0 +1.7.0 diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index f55ecc213..b68099cad 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,14 +1,10 @@ -import sys +import six from six.moves import copyreg -if sys.version_info[0] == 2: + +if six.PY2: from urlparse import urlparse - - # workaround for https://bugs.python.org/issue7904 - Python < 2.7 - if urlparse('s3://bucket/key').netloc != 'bucket': - from urlparse import uses_netloc - uses_netloc.append('s3') - + # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4 if urlparse('s3://bucket/key?key=value').query != 'key=value': from urlparse import uses_query diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 783d4c383..5ac20c0bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -28,9 +28,15 @@ if twisted_version >= (14, 0, 0): understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ - def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs): + def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs): super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) self._ssl_method = method + self.tls_verbose_logging = tls_verbose_logging + + @classmethod + def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs): + tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING') + return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs) def getCertificateOptions(self): # setting verify=True will require you to provide CAs @@ -56,7 +62,8 @@ if twisted_version >= (14, 0, 0): return self.getCertificateOptions().getContext() def creatorForNetloc(self, hostname, port): - return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext()) + return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), + verbose_logging=self.tls_verbose_logging) @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index c342d4ab1..806a537d4 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -59,7 +59,7 @@ class ReceivedDataProtocol(Protocol): def close(self): self.body.close() if self.filename else self.body.seek(0) -_CODE_RE = re.compile("\d+") +_CODE_RE = re.compile(r"\d+") class FTPDownloadHandler(object): diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index d875fb1e4..be7298531 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,7 +1,7 @@ """Download handlers for http and https schemes """ from twisted.internet import reactor -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, create_instance from scrapy.utils.python import to_unicode @@ -11,6 +11,7 @@ class HTTP10DownloadHandler(object): def __init__(self, settings): self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + self._settings = settings def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -21,7 +22,7 @@ class HTTP10DownloadHandler(object): def _connect(self, factory): host, port = to_unicode(factory.host), factory.port if factory.scheme == b'https': - return reactor.connectSSL(host, port, factory, - self.ClientContextFactory()) + client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None) + return reactor.connectSSL(host, port, factory, client_context_factory) else: return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0673188a1..2ccab2614 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -25,7 +25,7 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.core.downloader.tls import openssl_methods -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, create_instance from scrapy.utils.python import to_bytes, to_unicode from scrapy import twisted_version @@ -44,14 +44,15 @@ class HTTP11DownloadHandler(object): self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) # try method-aware context factory try: - self._contextFactory = self._contextFactoryClass(method=self._sslMethod) + self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None, + method=self._sslMethod) except TypeError: # use context factory defaults - self._contextFactory = self._contextFactoryClass() + self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None) msg = """ '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD).\ - Please upgrade your context factory class to handle it or ignore it.""" % ( + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') @@ -101,7 +102,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ - _responseMatcher = re.compile(b'HTTP/1\.. (?P\d{3})(?P.{,32})') + _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})') def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): @@ -479,10 +480,10 @@ class _ResponseReader(protocol.Protocol): return elif not self._fail_on_dataloss_warned: - logger.warn("Got data loss in %s. If you want to process broken " - "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" - " -- This message won't be shown in further requests", - self._txresponse.request.absoluteURI.decode()) + logger.warning("Got data loss in %s. If you want to process broken " + "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + " -- This message won't be shown in further requests", + self._txresponse.request.absoluteURI.decode()) self._fail_on_dataloss_warned = True self._finished.errback(reason) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index df8051182..74afb3f10 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -2,6 +2,7 @@ import logging from OpenSSL import SSL from scrapy import twisted_version +from scrapy.utils.ssl import x509name_to_string, get_temp_key_info logger = logging.getLogger(__name__) @@ -20,6 +21,7 @@ openssl_methods = { METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only } + if twisted_version >= (14, 0, 0): # ClientTLSOptions requires a recent-enough version of Twisted. # Not having ScrapyClientTLSOptions should not matter for older @@ -65,13 +67,39 @@ if twisted_version >= (14, 0, 0): Same as Twisted's private _sslverify.ClientTLSOptions, except that VerificationError, CertificateError and ValueError exceptions are caught, so that the connection is not closed, only - logging warnings. + logging warnings. Also, HTTPS connection parameters logging is added. """ + def __init__(self, hostname, ctx, verbose_logging=False): + super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + self.verbose_logging = verbose_logging + def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL_CB_HANDSHAKE_START: set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL_CB_HANDSHAKE_DONE: + if self.verbose_logging: + if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15 + if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0 + logger.debug('SSL connection to %s using protocol %s, cipher %s', + self._hostnameASCII, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) + else: + logger.debug('SSL connection to %s using cipher %s', + self._hostnameASCII, + connection.get_cipher_name(), + ) + server_cert = connection.get_peer_certificate() + logger.debug('SSL connection certificate: issuer "%s", subject "%s"', + x509name_to_string(server_cert.get_issuer()), + x509name_to_string(server_cert.get_subject()), + ) + key_info = get_temp_key_info(connection._ssl) + if key_info: + logger.debug('SSL temp key: %s', key_info) + try: verifyHostname(connection, self._hostnameASCII) except verification_errors as e: diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 1c89a0f9e..3a5890ed0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -95,7 +95,7 @@ class ScrapyHTTPPageGetter(HTTPClient): class ScrapyHTTPClientFactory(HTTPClientFactory): """Scrapy implementation of the HTTPClientFactory overwriting the - serUrl method to make use of our Url object that cache the parse + setUrl method to make use of our Url object that cache the parse result. """ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f573b2b7f..ded3c082b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -39,14 +39,15 @@ class Crawler(object): self.settings = settings.copy() self.spidercls.update_settings(self.settings) - d = dict(overridden_settings(self.settings)) - logger.info("Overridden settings: %(settings)r", {'settings': d}) - self.signals = SignalManager(self) self.stats = load_object(self.settings['STATS_CLASS'])(self) handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL')) logging.root.addHandler(handler) + + d = dict(overridden_settings(self.settings)) + logger.info("Overridden settings: %(settings)r", {'settings': d}) + if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings install_scrapy_root_handler(self.settings) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 200245210..c5a60d355 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -5,8 +5,8 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ import logging - -from six.moves.urllib import robotparser +import sys +import re from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import NotConfigured, IgnoreRequest @@ -14,6 +14,7 @@ from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.python import to_native_str +from scrapy.utils.misc import load_object logger = logging.getLogger(__name__) @@ -24,10 +25,13 @@ class RobotsTxtMiddleware(object): def __init__(self, crawler): if not crawler.settings.getbool('ROBOTSTXT_OBEY'): raise NotConfigured - + self._default_useragent = crawler.settings.get('USER_AGENT', 'Scrapy') self.crawler = crawler - self._useragent = crawler.settings.get('USER_AGENT') self._parsers = {} + self._parserimpl = load_object(crawler.settings.get('ROBOTSTXT_PARSER')) + + # check if parser dependencies are met, this should throw an error otherwise. + self._parserimpl.from_crawler(self.crawler, b'') @classmethod def from_crawler(cls, crawler): @@ -43,7 +47,8 @@ class RobotsTxtMiddleware(object): def process_request_2(self, rp, request, spider): if rp is None: return - if not rp.can_fetch(to_native_str(self._useragent), request.url): + useragent = request.headers.get(b'User-Agent', self._default_useragent) + if not rp.allowed(request.url, useragent): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) self.crawler.stats.inc_value('robotstxt/forbidden') @@ -62,13 +67,14 @@ class RobotsTxtMiddleware(object): meta={'dont_obey_robotstxt': True} ) dfd = self.crawler.engine.download(robotsreq, spider) - dfd.addCallback(self._parse_robots, netloc) + dfd.addCallback(self._parse_robots, netloc, spider) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) self.crawler.stats.inc_value('robotstxt/request_count') if isinstance(self._parsers[netloc], Deferred): d = Deferred() + def cb(result): d.callback(result) return result @@ -85,27 +91,10 @@ class RobotsTxtMiddleware(object): extra={'spider': spider}) return failure - def _parse_robots(self, response, netloc): + def _parse_robots(self, response, netloc, spider): self.crawler.stats.inc_value('robotstxt/response_count') - self.crawler.stats.inc_value( - 'robotstxt/response_status_count/{}'.format(response.status)) - rp = robotparser.RobotFileParser(response.url) - body = '' - if hasattr(response, 'text'): - body = response.text - else: # last effort try - try: - body = response.body.decode('utf-8') - except UnicodeDecodeError: - # If we found garbage, disregard it:, - # but keep the lookup cached (in self._parsers) - # Running rp.parse() will set rp state from - # 'disallow all' to 'allow any'. - self.crawler.stats.inc_value('robotstxt/unicode_error_count') - # stdlib's robotparser expects native 'str' ; - # with unicode input, non-ASCII encoded bytes decoding fails in Python2 - rp.parse(to_native_str(body).splitlines()) - + self.crawler.stats.inc_value('robotstxt/response_status_count/{}'.format(response.status)) + rp = self._parserimpl.from_crawler(self.crawler, response.body) rp_dfd = self._parsers[netloc] self._parsers[netloc] = rp rp_dfd.callback(rp) diff --git a/scrapy/item.py b/scrapy/item.py index 031b80a2d..9d4786788 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,16 +4,22 @@ Scrapy Item See documentation in docs/topics/item.rst """ -from pprint import pformat -from collections import MutableMapping -from copy import deepcopy - from abc import ABCMeta +from pprint import pformat +from copy import deepcopy +import collections + import six from scrapy.utils.trackref import object_ref +if six.PY2: + MutableMapping = collections.MutableMapping +else: + MutableMapping = collections.abc.MutableMapping + + class BaseItem(object_ref): """Base class for all scraped items.""" pass diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index 468aec2cc..2acdc8093 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -3,10 +3,13 @@ This module provides some commonly used processors for Item Loaders. See documentation in docs/topics/loaders.rst """ +try: + from collections import ChainMap +except ImportError: + from scrapy.utils.datatypes import MergeDict as ChainMap from scrapy.utils.misc import arg_to_iter -from scrapy.utils.datatypes import MergeDict -from .common import wrap_loader_context +from scrapy.loader.common import wrap_loader_context class MapCompose(object): @@ -18,7 +21,7 @@ class MapCompose(object): def __call__(self, value, loader_context=None): values = arg_to_iter(value) if loader_context: - context = MergeDict(loader_context, self.default_loader_context) + context = ChainMap(loader_context, self.default_loader_context) else: context = self.default_loader_context wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] @@ -45,7 +48,7 @@ class Compose(object): def __call__(self, value, loader_context=None): if loader_context: - context = MergeDict(loader_context, self.default_loader_context) + context = ChainMap(loader_context, self.default_loader_context) else: context = self.default_loader_context wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 65f347dcf..b4d6787ff 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -12,26 +12,40 @@ CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(ref class LogFormatter(object): """Class for generating log messages for different actions. - - All methods must return a dictionary listing the parameters ``level``, - ``msg`` and ``args`` which are going to be used for constructing the log - message when calling logging.log. + + All methods must return a dictionary listing the parameters ``level``, ``msg`` + and ``args`` which are going to be used for constructing the log message when + calling ``logging.log``. Dictionary keys for the method outputs: - * ``level`` should be the log level for that action, you can use those - from the python logging library: logging.DEBUG, logging.INFO, - logging.WARNING, logging.ERROR and logging.CRITICAL. - * ``msg`` should be a string that can contain different formatting - placeholders. This string, formatted with the provided ``args``, is - going to be the log message for that action. + * ``level`` is the log level for that action, you can use those from the + `python logging library `_ : + ``logging.DEBUG``, ``logging.INFO``, ``logging.WARNING``, ``logging.ERROR`` + and ``logging.CRITICAL``. + * ``msg`` should be a string that can contain different formatting placeholders. + This string, formatted with the provided ``args``, is going to be the long message + for that action. + * ``args`` should be a tuple or dict with the formatting placeholders for ``msg``. + The final log message is computed as ``msg % args``. - * ``args`` should be a tuple or dict with the formatting placeholders - for ``msg``. The final log message is computed as output['msg'] % - output['args']. + Here is an example on how to create a custom log formatter to lower the severity level of + the log message when an item is dropped from the pipeline:: + + class PoliteLogFormatter(logformatter.LogFormatter): + def dropped(self, item, exception, response, spider): + return { + 'level': logging.INFO, # lowering the level from logging.WARNING + 'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s", + 'args': { + 'exception': exception, + 'item': item, + } + } """ - + def crawled(self, request, response, spider): + """Logs a message when the crawler finds a webpage.""" request_flags = ' %s' % str(request.flags) if request.flags else '' response_flags = ' %s' % str(response.flags) if response.flags else '' return { @@ -40,7 +54,7 @@ class LogFormatter(object): 'args': { 'status': response.status, 'request': request, - 'request_flags' : request_flags, + 'request_flags': request_flags, 'referer': referer_str(request), 'response_flags': response_flags, # backward compatibility with Scrapy logformatter below 1.4 version @@ -49,6 +63,7 @@ class LogFormatter(object): } def scraped(self, item, response, spider): + """Logs a message when an item is scraped by a spider.""" if isinstance(response, Failure): src = response.getErrorMessage() else: @@ -63,6 +78,7 @@ class LogFormatter(object): } def dropped(self, item, exception, response, spider): + """Logs a message when an item is dropped while it is passing through the item pipeline.""" return { 'level': logging.WARNING, 'msg': DROPPEDMSG, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 2145e6d2b..ea06d2ae8 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -189,6 +189,19 @@ class S3FilesStore(object): 'X-Amz-Grant-Read': 'GrantRead', 'X-Amz-Grant-Read-ACP': 'GrantReadACP', 'X-Amz-Grant-Write-ACP': 'GrantWriteACP', + 'X-Amz-Object-Lock-Legal-Hold': 'ObjectLockLegalHoldStatus', + 'X-Amz-Object-Lock-Mode': 'ObjectLockMode', + 'X-Amz-Object-Lock-Retain-Until-Date': 'ObjectLockRetainUntilDate', + 'X-Amz-Request-Payer': 'RequestPayer', + 'X-Amz-Server-Side-Encryption': 'ServerSideEncryption', + 'X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id': 'SSEKMSKeyId', + 'X-Amz-Server-Side-Encryption-Context': 'SSEKMSEncryptionContext', + 'X-Amz-Server-Side-Encryption-Customer-Algorithm': 'SSECustomerAlgorithm', + 'X-Amz-Server-Side-Encryption-Customer-Key': 'SSECustomerKey', + 'X-Amz-Server-Side-Encryption-Customer-Key-Md5': 'SSECustomerKeyMD5', + 'X-Amz-Storage-Class': 'StorageClass', + 'X-Amz-Tagging': 'Tagging', + 'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation', }) extra = {} for key, value in six.iteritems(headers): diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py new file mode 100644 index 000000000..4bfb275fd --- /dev/null +++ b/scrapy/robotstxt.py @@ -0,0 +1,112 @@ +import sys +import logging +from abc import ABCMeta, abstractmethod +from six import with_metaclass + +from scrapy.utils.python import to_native_str, to_unicode + +logger = logging.getLogger(__name__) + + +class RobotParser(with_metaclass(ABCMeta)): + @classmethod + @abstractmethod + def from_crawler(cls, crawler, robotstxt_body): + """Parse the content of a robots.txt_ file as bytes. This must be a class method. + It must return a new instance of the parser backend. + + :param crawler: crawler which made the request + :type crawler: :class:`~scrapy.crawler.Crawler` instance + + :param robotstxt_body: content of a robots.txt_ file. + :type robotstxt_body: bytes + """ + pass + + @abstractmethod + def allowed(self, url, user_agent): + """Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``. + + :param url: Absolute URL + :type url: string + + :param user_agent: User agent + :type user_agent: string + """ + pass + + +class PythonRobotParser(RobotParser): + def __init__(self, robotstxt_body, spider): + from six.moves.urllib_robotparser import RobotFileParser + self.spider = spider + try: + robotstxt_body = to_native_str(robotstxt_body) + except UnicodeDecodeError: + # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. + # Switch to 'allow all' state. + logger.warning("Failure while parsing robots.txt using %(parser)s." + " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.", + {'parser': "RobotFileParser"}, + exc_info=sys.exc_info(), + extra={'spider': self.spider}) + robotstxt_body = '' + self.rp = RobotFileParser() + self.rp.parse(robotstxt_body.splitlines()) + + @classmethod + def from_crawler(cls, crawler, robotstxt_body): + spider = None if not crawler else crawler.spider + o = cls(robotstxt_body, spider) + return o + + def allowed(self, url, user_agent): + user_agent = to_native_str(user_agent) + url = to_native_str(url) + return self.rp.can_fetch(user_agent, url) + + +class ReppyRobotParser(RobotParser): + def __init__(self, robotstxt_body, spider): + from reppy.robots import Robots + self.spider = spider + self.rp = Robots.parse('', robotstxt_body) + + @classmethod + def from_crawler(cls, crawler, robotstxt_body): + spider = None if not crawler else crawler.spider + o = cls(robotstxt_body, spider) + return o + + def allowed(self, url, user_agent): + return self.rp.allowed(url, user_agent) + + +class RerpRobotParser(RobotParser): + def __init__(self, robotstxt_body, spider): + from robotexclusionrulesparser import RobotExclusionRulesParser + self.spider = spider + self.rp = RobotExclusionRulesParser() + try: + robotstxt_body = robotstxt_body.decode('utf-8') + except UnicodeDecodeError: + # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it. + # Switch to 'allow all' state. + logger.warning("Failure while parsing robots.txt using %(parser)s." + " File either contains garbage or is in an encoding other than UTF-8, treating it as an empty file.", + {'parser': "RobotExclusionRulesParser"}, + exc_info=sys.exc_info(), + extra={'spider': self.spider}) + robotstxt_body = '' + self.rp.parse(robotstxt_body) + + @classmethod + def from_crawler(cls, crawler, robotstxt_body): + spider = None if not crawler else crawler.spider + o = cls(robotstxt_body, spider) + return o + + def allowed(self, url, user_agent): + user_agent = to_unicode(user_agent) + url = to_unicode(url) + return self.rp.is_allowed(user_agent, url) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 14c93bef2..f28c7940d 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,11 +1,17 @@ import six import json import copy -from collections import MutableMapping +import collections from importlib import import_module from pprint import pformat -from . import default_settings +from scrapy.settings import default_settings + + +if six.PY2: + MutableMapping = collections.MutableMapping +else: + MutableMapping = collections.abc.MutableMapping SETTINGS_PRIORITIES = { diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d17eb3125..81fee543f 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -87,6 +87,7 @@ DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClien DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform, # also allowing negotiation +DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False DOWNLOADER_MIDDLEWARES = {} @@ -244,6 +245,7 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False +ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser' SCHEDULER = 'scrapy.core.scheduler.Scheduler' SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue' diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index fbd297340..fb7ca3310 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -4,7 +4,10 @@ import numbers from operator import itemgetter import six -from six.moves.configparser import SafeConfigParser +if six.PY2: + from ConfigParser import SafeConfigParser as ConfigParser +else: + from configparser import ConfigParser from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath @@ -92,9 +95,9 @@ def init_env(project='default', set_syspath=True): def get_config(use_closest=True): - """Get Scrapy config file as a SafeConfigParser""" + """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) - cfg = SafeConfigParser() + cfg = ConfigParser() cfg.read(sources) return cfg diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index eb373c501..df2b99c28 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,13 +6,20 @@ This module must not depend on any module outside the Standard Library. """ import copy -import six +import collections import warnings -from collections import OrderedDict, Mapping + +import six from scrapy.exceptions import ScrapyDeprecationWarning +if six.PY2: + Mapping = collections.Mapping +else: + Mapping = collections.abc.Mapping + + class MultiValueDictKeyError(KeyError): def __init__(self, *args, **kwargs): warnings.warn( @@ -245,6 +252,13 @@ class MergeDict(object): first occurrence will be used. """ def __init__(self, *dicts): + if not six.PY2: + warnings.warn( + "scrapy.utils.datatypes.MergeDict is deprecated in favor " + "of collections.ChainMap (introduced in Python 3.3)", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.dicts = dicts def __getitem__(self, key): @@ -289,7 +303,7 @@ class MergeDict(object): return self.__copy__() -class LocalCache(OrderedDict): +class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. Older items expires first. diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py new file mode 100644 index 000000000..e54232abd --- /dev/null +++ b/scrapy/utils/ssl.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +import OpenSSL._util as pyOpenSSLutil + +from scrapy.utils.python import to_native_str + + +def ffi_buf_to_string(buf): + return to_native_str(pyOpenSSLutil.ffi.string(buf)) + + +def x509name_to_string(x509name): + # from OpenSSL.crypto.X509Name.__repr__ + result_buffer = pyOpenSSLutil.ffi.new("char[]", 512) + pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer)) + + return ffi_buf_to_string(result_buffer) + + +def get_temp_key_info(ssl_object): + if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2 + return None + + # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() + temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") + if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): + return None + temp_key = temp_key_p[0] + if temp_key == pyOpenSSLutil.ffi.NULL: + return None + temp_key = pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free) + key_info = [] + key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key) + if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA: + key_info.append('RSA') + elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH: + key_info.append('DH') + elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC: + key_info.append('ECDH') + ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key) + ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free) + nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key)) + cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid) + if cname == pyOpenSSLutil.ffi.NULL: + cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid) + key_info.append(ffi_buf_to_string(cname)) + else: + key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) + key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)) + return ', '.join(key_info) diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 1d7bd006c..615372fc8 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -18,7 +18,7 @@ def render_templatefile(path, **kwargs): os.remove(path) -CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]') +CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') def string_camelcase(string): """ Convert a word to its CamelCase version and remove invalid chars diff --git a/setup.py b/setup.py index 4dc6d18c1..37892cfbf 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', @@ -63,19 +62,21 @@ setup( 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', install_requires=[ - 'Twisted>=13.1.0;python_version!="3.4"', - 'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"', - 'w3lib>=1.17.0', - 'queuelib', - 'lxml', - 'pyOpenSSL', - 'cssselect>=0.9', - 'six>=1.5.2', - 'parsel>=1.5', + 'Twisted>=16.0.0;python_version=="2.7"', + 'Twisted>=17.9.0;python_version>="3.5"', + 'cryptography>=2.0', + 'cssselect>=0.9.1', + 'lxml>=3.5.0', + 'parsel>=1.5.0', 'PyDispatcher>=2.0.5', - 'service_identity', + 'pyOpenSSL>=16.2.0', + 'queuelib>=1.4.2', + 'service_identity>=16.0.0', + 'six>=1.10.0', + 'w3lib>=1.17.0', + 'zope.interface>=4.1.3', ], extras_require=extras_require, ) diff --git a/tests/__init__.py b/tests/__init__.py index a54367f8c..9c9e35c35 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -35,3 +35,18 @@ def get_testdata(*paths): path = os.path.join(tests_datadir, *paths) with open(path, 'rb') as f: return f.read() + + +# FIXME: delete after dropping py2 support +# Monkey patch the unittest module to prevent the +# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex +import six +if six.PY2: + import unittest + import twisted.trial.unittest + if not getattr(unittest.TestCase, 'assertRegex', None): + unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches + if not getattr(unittest.TestCase, 'assertRaisesRegex', None): + unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None): + twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp diff --git a/tests/constraints.txt b/tests/constraints.txt index e59e68b3f..5655ac2d3 100644 --- a/tests/constraints.txt +++ b/tests/constraints.txt @@ -1,2 +1 @@ -Twisted!=18.4.0 -lxml!=4.2.2 \ No newline at end of file +Twisted!=18.4.0 \ No newline at end of file diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index be809b151..f621eb4eb 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -1,14 +1,15 @@ # Tests requirements -mock +brotlipy +jmespath mitmproxy==0.10.1 +mock netlib==0.10.1 pytest pytest-cov pytest-twisted pytest-xdist -jmespath -brotlipy testfixtures + # optional for shell wrapper tests bpython ipython<6.0 diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index ed7bf0be0..cb67bc40e 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,13 +1,14 @@ +# Tests requirements +jmespath +leveldb; sys_platform != "win32" pytest pytest-cov pytest-twisted pytest-xdist testfixtures -jmespath -leveldb; sys_platform != "win32" -botocore + # optional for shell wrapper tests bpython -ipython brotlipy +ipython pywin32; sys_platform == "win32" diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index c18a6ce9f..98e415ad3 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -170,7 +170,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find callback""", _textmode(stderr)) @defer.inlineCallbacks @@ -195,7 +195,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) @defer.inlineCallbacks @@ -203,7 +203,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") @defer.inlineCallbacks def test_crawlspider_no_matching_rule(self): @@ -211,5 +211,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index d2151e10e..efef4192c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -8,6 +8,7 @@ try: except ImportError: import mock +from testfixtures import LogCapture from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory from twisted.python.filepath import FilePath @@ -498,6 +499,24 @@ class Http11TestCase(HttpTestCase): class Https11TestCase(Http11TestCase): scheme = 'https' + tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"' + + @defer.inlineCallbacks + def test_tls_logging(self): + download_handler = self.download_handler_cls(Settings({ + 'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True, + })) + try: + with LogCapture() as log_capture: + request = Request(self.getURL('file')) + d = download_handler.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + log_capture.check_present(('scrapy.core.downloader.tls', 'DEBUG', self.tls_log_message)) + finally: + yield download_handler.close() + class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -518,6 +537,7 @@ class Https11InvalidDNSId(Https11TestCase): super(Https11InvalidDNSId, self).setUp() self.host = '127.0.0.1' + class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -529,6 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase): from service_identity.exceptions import CertificateError except ImportError: raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' super(Https11InvalidDNSPattern, self).setUp() diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 17801e502..04884fb78 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -13,7 +13,7 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): - cookievaleq = lambda cv: re.split(';\s*', cv.decode('latin1')) + cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1')) return self.assertEqual( sorted(cookievaleq(first)), sorted(cookievaleq(second)), msg) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 60306eacb..79f172848 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import re from twisted.internet import reactor, error from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -11,6 +10,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.settings import Settings from tests import mock +from tests.test_robotstxt_interface import rerp_available, reppy_available class RobotsTxtMiddlewareTest(unittest.TestCase): @@ -31,19 +31,18 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = re.sub(b'^\s+(?m)', b'', u''' - User-Agent: * - Disallow: /admin/ - Disallow: /static/ - - # taken from https://en.wikipedia.org/robots.txt - Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: - Disallow: /wiki/Käyttäjä: - - User-Agent: UnicödeBöt - Disallow: /some/randome/page.html - '''.encode('utf-8')) + ROBOTS = u""" +User-Agent: * +Disallow: /admin/ +Disallow: /static/ +# taken from https://en.wikipedia.org/robots.txt +Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: +Disallow: /wiki/Käyttäjä: +User-Agent: UnicödeBöt +Disallow: /some/randome/page.html +""".encode('utf-8') response = TextResponse('http://site.local/robots.txt', body=ROBOTS) + def return_response(request, spider): deferred = Deferred() reactor.callFromThread(deferred.callback, response) @@ -80,6 +79,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt', body=b'GIF89a\xd3\x00\xfe\x00\xa2') + def return_response(request, spider): deferred = Deferred() reactor.callFromThread(deferred.callback, response) @@ -102,6 +102,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) response = Response('http://site.local/robots.txt') + def return_response(request, spider): deferred = Deferred() reactor.callFromThread(deferred.callback, response) @@ -121,6 +122,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def test_robotstxt_error(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') + def return_failure(request, spider): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(err)) @@ -136,6 +138,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def test_robotstxt_immediate_error(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) err = error.DNSLookupError('Robotstxt address not found') + def immediate_failure(request, spider): deferred = Deferred() deferred.errback(failure.Failure(err)) @@ -147,6 +150,7 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def test_ignore_robotstxt_request(self): self.crawler.settings.set('ROBOTSTXT_OBEY', True) + def ignore_request(request, spider): deferred = Deferred() reactor.callFromThread(deferred.errback, failure.Failure(IgnoreRequest())) @@ -170,3 +174,21 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): spider = None # not actually used return self.assertFailure(maybeDeferred(middleware.process_request, request, spider), IgnoreRequest) + + +class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): + if not rerp_available(): + skip = "Rerp parser is not installed" + + def setUp(self): + super(RobotsTxtMiddlewareWithRerpTest, self).setUp() + self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser') + + +class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): + if not reppy_available(): + skip = "Reppy parser is not installed" + + def setUp(self): + super(RobotsTxtMiddlewareWithReppyTest, self).setUp() + self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser') diff --git a/tests/test_engine.py b/tests/test_engine.py index 856465161..30150391a 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -40,9 +40,9 @@ class TestSpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] - itemurl_re = re.compile("item\d+.html") - name_re = re.compile("

(.*?)

", re.M) - price_re = re.compile(">Price: \$(.*?)<", re.M) + itemurl_re = re.compile(r"item\d+.html") + name_re = re.compile(r"

(.*?)

", re.M) + price_re = re.compile(r">Price: \$(.*?)<", re.M) item_cls = TestItem diff --git a/tests/test_exporters.py b/tests/test_exporters.py index cd72c661a..0046c5666 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -103,7 +103,7 @@ class PythonItemExporterTest(BaseItemExporterTest): return PythonItemExporter(binary=False, **kwargs) def test_invalid_option(self): - with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"): + with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): PythonItemExporter(invalid_option='something') def test_nested_item(self): diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 504937295..69d906fbf 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -147,11 +147,11 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h1.getlist('hey'), [b'5']) def test_invalid_value(self): - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers, {'foo': object()}) - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers().__setitem__, 'foo', object()) - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers().setdefault, 'foo', object()) - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers().setlist, 'foo', [object()]) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers, {'foo': object()}) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers().__setitem__, 'foo', object()) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers().setdefault, 'foo', object()) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers().setlist, 'foo', [object()]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 53d18d4aa..952e208de 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -989,9 +989,9 @@ class FormRequestTest(RequestTest): xpath = u"//form[@name='\u03b1']" encoded = xpath if six.PY3 else xpath.encode('unicode_escape') - self.assertRaisesRegexp(ValueError, re.escape(encoded), - self.request_class.from_response, - response, formxpath=xpath) + self.assertRaisesRegex(ValueError, re.escape(encoded), + self.request_class.from_response, + response, formxpath=xpath) def test_from_response_button_submit(self): response = _buildresponse( diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 3b90e3dac..cd5c3486e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -135,9 +135,9 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com", body=b'hello') if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegexp(AttributeError, msg, getattr, r, 'text') - self.assertRaisesRegexp(NotSupported, msg, r.css, 'body') - self.assertRaisesRegexp(NotSupported, msg, r.xpath, '//body') + self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text') + self.assertRaisesRegex(NotSupported, msg, r.css, 'body') + self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body') else: r.text r.css('body') @@ -425,13 +425,13 @@ class TextResponseTest(BaseResponseTest): def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegexp(ValueError, 'SelectorList', - resp.follow, resp.css('a')) + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) def test_follow_selector_invalid(self): resp = self._links_response() - self.assertRaisesRegexp(ValueError, 'Unsupported', - resp.follow, resp.xpath('count(//div)')[0]) + self.assertRaisesRegex(ValueError, 'Unsupported', + resp.follow, resp.xpath('count(//div)')[0]) def test_follow_selector_attribute(self): resp = self._links_response() @@ -443,8 +443,8 @@ class TextResponseTest(BaseResponseTest): url='http://example.com', body=b'click me', ) - self.assertRaisesRegexp(ValueError, 'no href', - resp.follow, resp.css('a')[0]) + self.assertRaisesRegex(ValueError, 'no href', + resp.follow, resp.css('a')[0]) def test_follow_whitespace_selector(self): resp = self.response_class( diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index c9cd629f4..d96e259f6 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -288,7 +288,7 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') def process_value(value): - m = re.search("javascript:goToPage\('(.*?)'", value) + m = re.search(r"javascript:goToPage\('(.*?)'", value) if m: return m.group(1) diff --git a/tests/test_loader.py b/tests/test_loader.py index ce0fa0701..2725b001a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -691,7 +691,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) - l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)') + l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)') self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py new file mode 100644 index 000000000..2819786b5 --- /dev/null +++ b/tests/test_robotstxt_interface.py @@ -0,0 +1,142 @@ +# coding=utf-8 +from twisted.trial import unittest +from scrapy.utils.python import to_native_str + + +def reppy_available(): + # check if reppy parser is installed + try: + from reppy.robots import Robots + except ImportError: + return False + return True + + +def rerp_available(): + # check if robotexclusionrulesparser is installed + try: + from robotexclusionrulesparser import RobotExclusionRulesParser + except ImportError: + return False + return True + + +class BaseRobotParserTest: + def _setUp(self, parser_cls): + self.parser_cls = parser_cls + + def test_allowed(self): + robotstxt_robotstxt_body = ("User-agent: * \n" + "Disallow: /disallowed \n" + "Allow: /allowed \n" + "Crawl-delay: 10".encode('utf-8')) + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body) + self.assertTrue(rp.allowed("https://www.site.local/allowed", "*")) + self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*")) + + def test_allowed_wildcards(self): + robotstxt_robotstxt_body = """User-agent: first + Disallow: /disallowed/*/end$ + + User-agent: second + Allow: /*allowed + Disallow: / + """.encode('utf-8') + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body) + + self.assertTrue(rp.allowed("https://www.site.local/disallowed", "first")) + self.assertFalse(rp.allowed("https://www.site.local/disallowed/xyz/end", "first")) + self.assertFalse(rp.allowed("https://www.site.local/disallowed/abc/end", "first")) + self.assertTrue(rp.allowed("https://www.site.local/disallowed/xyz/endinglater", "first")) + + self.assertTrue(rp.allowed("https://www.site.local/allowed", "second")) + self.assertTrue(rp.allowed("https://www.site.local/is_still_allowed", "second")) + self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second")) + + def test_length_based_precedence(self): + robotstxt_robotstxt_body = ("User-agent: * \n" + "Disallow: / \n" + "Allow: /page".encode('utf-8')) + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body) + self.assertTrue(rp.allowed("https://www.site.local/page", "*")) + + def test_order_based_precedence(self): + robotstxt_robotstxt_body = ("User-agent: * \n" + "Disallow: / \n" + "Allow: /page".encode('utf-8')) + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body) + self.assertFalse(rp.allowed("https://www.site.local/page", "*")) + + def test_empty_response(self): + """empty response should equal 'allow all'""" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b'') + self.assertTrue(rp.allowed("https://site.local/", "*")) + self.assertTrue(rp.allowed("https://site.local/", "chrome")) + self.assertTrue(rp.allowed("https://site.local/index.html", "*")) + self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + + def test_garbage_response(self): + """garbage response should be discarded, equal 'allow all'""" + robotstxt_robotstxt_body = b'GIF89a\xd3\x00\xfe\x00\xa2' + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body) + self.assertTrue(rp.allowed("https://site.local/", "*")) + self.assertTrue(rp.allowed("https://site.local/", "chrome")) + self.assertTrue(rp.allowed("https://site.local/index.html", "*")) + self.assertTrue(rp.allowed("https://site.local/disallowed", "*")) + + def test_unicode_url_and_useragent(self): + robotstxt_robotstxt_body = u""" + User-Agent: * + Disallow: /admin/ + Disallow: /static/ + # taken from https://en.wikipedia.org/robots.txt + Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: + Disallow: /wiki/Käyttäjä: + + User-Agent: UnicödeBöt + Disallow: /some/randome/page.html""".encode('utf-8') + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_robotstxt_body) + self.assertTrue(rp.allowed("https://site.local/", "*")) + self.assertFalse(rp.allowed("https://site.local/admin/", "*")) + self.assertFalse(rp.allowed("https://site.local/static/", "*")) + self.assertTrue(rp.allowed("https://site.local/admin/", u"UnicödeBöt")) + self.assertFalse(rp.allowed("https://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:", "*")) + self.assertFalse(rp.allowed(u"https://site.local/wiki/Käyttäjä:", "*")) + self.assertTrue(rp.allowed("https://site.local/some/randome/page.html", "*")) + self.assertFalse(rp.allowed("https://site.local/some/randome/page.html", u"UnicödeBöt")) + + +class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): + def setUp(self): + from scrapy.robotstxt import PythonRobotParser + super(PythonRobotParserTest, self)._setUp(PythonRobotParser) + + def test_length_based_precedence(self): + raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.") + + def test_allowed_wildcards(self): + raise unittest.SkipTest("RobotFileParser does not support wildcards.") + + +class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): + if not reppy_available(): + skip = "Reppy parser is not installed" + + def setUp(self): + from scrapy.robotstxt import ReppyRobotParser + super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser) + + def test_order_based_precedence(self): + raise unittest.SkipTest("Rerp does not support order based directives precedence.") + + +class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): + if not rerp_available(): + skip = "Rerp parser is not installed" + + def setUp(self): + from scrapy.robotstxt import RerpRobotParser + super(RerpRobotParserTest, self)._setUp(RerpRobotParser) + + def test_length_based_precedence(self): + raise unittest.SkipTest("Rerp does not support length based directives precedence.") diff --git a/tests/test_selector.py b/tests/test_selector.py index 210e3256c..b2565dd78 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -85,5 +85,5 @@ class SelectorTestCase(unittest.TestCase): x.__class__.__name__ def test_selector_bad_args(self): - with self.assertRaisesRegexp(ValueError, 'received both response and text'): + with self.assertRaisesRegex(ValueError, 'received both response and text'): Selector(TextResponse(url='http://example.com', body=b''), text=u'') diff --git a/tests/test_spider.py b/tests/test_spider.py index 1197ea653..e81e6d5f9 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -595,5 +595,5 @@ class NoParseMethodSpiderTest(unittest.TestCase): resp = TextResponse(url="http://www.example.com/random_url", body=text) exc_msg = 'Spider.parse callback is not defined' - with self.assertRaisesRegexp(NotImplementedError, exc_msg): + with self.assertRaisesRegex(NotImplementedError, exc_msg): spider.parse(resp) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 1cd59b99a..106da798c 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -84,8 +84,8 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.spider1' runner = CrawlerRunner({'SPIDER_MODULES': [module]}) - self.assertRaisesRegexp(KeyError, 'Spider not found', - runner.create_crawler, 'spider2') + self.assertRaisesRegex(KeyError, 'Spider not found', + runner.create_crawler, 'spider2') crawler = runner.create_crawler('spider1') self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 5b83869b8..535095b8d 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,11 +1,18 @@ import copy import unittest -from collections import Mapping, MutableMapping + +import six +if six.PY2: + from collections import Mapping, MutableMapping +else: + from collections.abc import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, SequenceExclude + __doctests__ = ['scrapy.utils.datatypes'] + class CaselessDictTest(unittest.TestCase): def test_init_dict(self): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index c2b9fc176..e6588055c 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -233,8 +233,8 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate ([ - ('C:\absolute\path\to\a\file.html', 'file://', +for k, args in enumerate([ + (r'C:\absolute\path\to\a\file.html', 'file://', 'Windows filepath are not supported for scrapy shell'), ], start=1): t_method = create_skipped_scheme_t(args) diff --git a/tox.ini b/tox.ini index 157a8b3ed..c3502c2ca 100644 --- a/tox.ini +++ b/tox.ini @@ -11,10 +11,10 @@ deps = -ctests/constraints.txt -rrequirements-py2.txt # Extras - botocore + botocore>=1.3.23 google-cloud-storage - Pillow != 3.0.0 leveldb + Pillow>=3.4.2 -rtests/requirements-py2.txt passenv = S3_TEST_FILE_URI @@ -25,72 +25,74 @@ passenv = commands = py.test --cov=scrapy --cov-report= {posargs:scrapy tests} -[testenv:trusty] +[testenv:py27-pinned] basepython = python2.7 deps = - pyOpenSSL==0.13 - lxml==3.3.3 - Twisted==13.2.0 - boto==2.20.1 - Pillow==2.3.0 + -ctests/constraints.txt + cryptography==2.0 cssselect==0.9.1 - zope.interface==4.0.5 + lxml==3.5.0 + parsel==1.5.0 + PyDispatcher==2.0.5 + pyOpenSSL==16.2.0 + queuelib==1.4.2 + service_identity==16.0.0 + six==1.10.0 + Twisted==16.0.0 + w3lib==1.17.0 + zope.interface==4.1.3 -rtests/requirements-py2.txt - -[testenv:jessie] -# https://packages.debian.org/en/jessie/python/ -# https://packages.debian.org/en/jessie/zope/ -basepython = python2.7 -deps = - cryptography==0.6.1 - pyOpenSSL==0.14 - lxml==3.4.0 - Twisted==14.0.2 - boto==2.34.0 - Pillow==2.6.1 - cssselect==0.9.1 - zope.interface==4.1.1 - -rtests/requirements-py2.txt -# Not used directly but allows boto GCE plugins to load. -# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 - google-compute-engine==2.8.12 - -[testenv:trunk] -basepython = python2.7 -commands = - pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib - pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib - py.test --cov=scrapy --cov-report= {posargs:scrapy tests} + # Extras + botocore==1.3.23 + Pillow==3.4.2 [testenv:pypy] basepython = pypy commands = py.test {posargs:scrapy tests} -[testenv:py34] -basepython = python3.4 +[testenv:py35] +basepython = python3.5 deps = -ctests/constraints.txt -rrequirements-py3.txt - # Extras - Pillow -rtests/requirements-py3.txt + # Extras + botocore>=1.3.23 + Pillow>=3.4.2 -[testenv:py35] +[testenv:py35-pinned] basepython = python3.5 -deps = {[testenv:py34]deps} +deps = + -ctests/constraints.txt + cryptography==2.0 + cssselect==0.9.1 + lxml==3.5.0 + parsel==1.5.0 + PyDispatcher==2.0.5 + pyOpenSSL==16.2.0 + queuelib==1.4.2 + service_identity==16.0.0 + six==1.10.0 + Twisted==17.9.0 + w3lib==1.17.0 + zope.interface==4.1.3 + -rtests/requirements-py3.txt + # Extras + botocore==1.3.23 + Pillow==3.4.2 [testenv:py36] basepython = python3.6 -deps = {[testenv:py34]deps} +deps = {[testenv:py35]deps} [testenv:py37] basepython = python3.7 -deps = {[testenv:py34]deps} +deps = {[testenv:py35]deps} [testenv:pypy3] basepython = pypy3 -deps = {[testenv:py34]deps} +deps = {[testenv:py35]deps} commands = py.test {posargs:scrapy tests} @@ -116,3 +118,17 @@ changedir = {[docs]changedir} deps = {[docs]deps} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck + +[testenv:py37-extra-deps] +basepython = python3.7 +deps = + {[testenv:py35]deps} + reppy + robotexclusionrulesparser + +[testenv:py27-extra-deps] +basepython = python2.7 +deps = + {[testenv]deps} + reppy + robotexclusionrulesparser