From 9578f490991fbefc4ad643588c87862a57a9f032 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 9 Sep 2019 07:36:55 +0000 Subject: [PATCH 01/16] use protego as a default robots.txt parser --- docs/topics/downloader-middleware.rst | 17 ++++++++--------- requirements-py2.txt | 1 + requirements-py3.txt | 1 + scrapy/settings/default_settings.py | 2 +- tox.ini | 2 -- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0845ef6e4..be96425cf 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -513,7 +513,7 @@ defines the methods described below. .. method:: open_spider(spider) - This method gets called after a spider has been opened for crawling. It handles + This method gets called after a spider has been opened for crawling. It handles the :signal:`open_spider ` signal. :param spider: the spider which has been opened @@ -521,8 +521,8 @@ defines the methods described below. .. method:: close_spider(spider) - This method gets called after a spider has been closed. It handles - the :signal:`close_spider ` signal. + This method gets called after a spider has been closed. It handles + the :signal:`close_spider ` signal. :param spider: the spider which has been closed :type spider: :class:`~scrapy.spiders.Spider` object @@ -1020,10 +1020,10 @@ the request will be ignored by this middleware even if RobotFileParser ~~~~~~~~~~~~~~~ -`RobotFileParser `_ is -Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's +`RobotFileParser `_ is +Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's 1996 draft specification `_. It lacks -support for wildcard matching. Scrapy uses this parser by default. +support for wildcard matching. In order to use this parser, set: @@ -1074,13 +1074,12 @@ Protego parser `Protego `_ is a pure-Python robots.txt_ parser. The parser is fully compliant with `Google's Robots.txt Specification -`_ hence supports wildcard +`_ hence supports wildcard matching, and uses the length based rule similar to `Reppy `_. +Scrapy uses this parser by default. In order to use this parser: -* Install `Protego `_ by running ``pip install protego`` - * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ProtegoRobotParser`` diff --git a/requirements-py2.txt b/requirements-py2.txt index 9e6944240..61176bdba 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -15,3 +15,4 @@ service_identity>=16.0.0 six>=1.10.0 Twisted>=16.0.0 zope.interface>=4.1.3 +protego diff --git a/requirements-py3.txt b/requirements-py3.txt index cd183a525..61e2e32d8 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -15,3 +15,4 @@ lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 zope.interface>=4.1.3 +protego diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 05ab4b628..9c22999cb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -246,7 +246,7 @@ RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False -ROBOTSTXT_PARSER = 'scrapy.robotstxt.PythonRobotParser' +ROBOTSTXT_PARSER = 'scrapy.robotstxt.ProtegoRobotParser' ROBOTSTXT_USER_AGENT = None SCHEDULER = 'scrapy.core.scheduler.Scheduler' diff --git a/tox.ini b/tox.ini index cc845faf1..c3502c2ca 100644 --- a/tox.ini +++ b/tox.ini @@ -125,7 +125,6 @@ deps = {[testenv:py35]deps} reppy robotexclusionrulesparser - protego [testenv:py27-extra-deps] basepython = python2.7 @@ -133,4 +132,3 @@ deps = {[testenv]deps} reppy robotexclusionrulesparser - protego \ No newline at end of file From 7af8c76649caf772627a189bb2d88c2d8fd620a2 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 9 Sep 2019 08:10:09 +0000 Subject: [PATCH 02/16] add pinned versions --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index c3502c2ca..fdd227d02 100644 --- a/tox.ini +++ b/tox.ini @@ -33,6 +33,7 @@ deps = cssselect==0.9.1 lxml==3.5.0 parsel==1.5.0 + Protego=0.1.15 PyDispatcher==2.0.5 pyOpenSSL==16.2.0 queuelib==1.4.2 @@ -69,6 +70,7 @@ deps = cssselect==0.9.1 lxml==3.5.0 parsel==1.5.0 + Protego=0.1.15 PyDispatcher==2.0.5 pyOpenSSL==16.2.0 queuelib==1.4.2 From e418554c21cdd9da87b914472e32668cc41d7e87 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 9 Sep 2019 08:12:32 +0000 Subject: [PATCH 03/16] use proper equal --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fdd227d02..ffe7360d3 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ deps = cssselect==0.9.1 lxml==3.5.0 parsel==1.5.0 - Protego=0.1.15 + Protego==0.1.15 PyDispatcher==2.0.5 pyOpenSSL==16.2.0 queuelib==1.4.2 @@ -70,7 +70,7 @@ deps = cssselect==0.9.1 lxml==3.5.0 parsel==1.5.0 - Protego=0.1.15 + Protego==0.1.15 PyDispatcher==2.0.5 pyOpenSSL==16.2.0 queuelib==1.4.2 From 38828d3fd45e7846ef3556d384e4c1fdc7a0c8ba Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 9 Sep 2019 17:04:13 +0300 Subject: [PATCH 04/16] Update docs/topics/downloader-middleware.rst Co-Authored-By: elacuesta --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index be96425cf..192bfd19a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1021,7 +1021,7 @@ RobotFileParser ~~~~~~~~~~~~~~~ `RobotFileParser `_ is -Python's inbuilt robots.txt_ parser. The parser is fully compliant with `Martijn Koster's +Python's built-in robots.txt_ parser. The parser is fully compliant with `Martijn Koster's 1996 draft specification `_. It lacks support for wildcard matching. From 7b33fa58fa46ae7fb96cb40d82ecf98eafcbd49d Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 9 Sep 2019 17:04:27 +0300 Subject: [PATCH 05/16] Update requirements-py2.txt Co-Authored-By: elacuesta --- requirements-py2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index 61176bdba..c865cbaef 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -15,4 +15,4 @@ service_identity>=16.0.0 six>=1.10.0 Twisted>=16.0.0 zope.interface>=4.1.3 -protego +protego>=0.1.15 From db202487f06a1710b32d991220dbd6656da8b2a0 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 9 Sep 2019 14:05:45 +0000 Subject: [PATCH 06/16] newer version of protego and move up to top --- requirements-py2.txt | 2 +- requirements-py3.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements-py2.txt b/requirements-py2.txt index c865cbaef..dde8d1c9c 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -1,6 +1,7 @@ parsel>=1.5.0 PyDispatcher>=2.0.5 w3lib>=1.17.0 +protego>=0.1.15 pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" @@ -15,4 +16,3 @@ service_identity>=16.0.0 six>=1.10.0 Twisted>=16.0.0 zope.interface>=4.1.3 -protego>=0.1.15 diff --git a/requirements-py3.txt b/requirements-py3.txt index 61e2e32d8..2c98e6f6d 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -2,6 +2,7 @@ parsel>=1.5.0 PyDispatcher>=2.0.5 Twisted>=17.9.0 w3lib>=1.17.0 +protego>=0.1.15 pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" @@ -15,4 +16,3 @@ lxml>=3.5.0 service_identity>=16.0.0 six>=1.10.0 zope.interface>=4.1.3 -protego From 6bd88711f2f5946bce6c72d46cd8a5ad3ce4ce86 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 10 Sep 2019 08:55:37 +0000 Subject: [PATCH 07/16] update documentation --- docs/topics/settings.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 943ba13ee..75e0af63b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -951,7 +951,7 @@ LOGSTATS_INTERVAL Default: ``60.0`` -The interval (in seconds) between each logging printout of the stats +The interval (in seconds) between each logging printout of the stats by :class:`~scrapy.extensions.logstats.LogStats`. .. setting:: MEMDEBUG_ENABLED @@ -1165,7 +1165,7 @@ If enabled, Scrapy will respect robots.txt policies. For more information see ROBOTSTXT_PARSER ---------------- -Default: ``'scrapy.robotstxt.PythonRobotParser'`` +Default: ``'scrapy.robotstxt.ProtegoRobotParser'`` The parser backend to use for parsing ``robots.txt`` files. For more information see :ref:`topics-dlmw-robots`. From c7f2bdfdbed0e82a385b05ea080768d38d6626cd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 10 Sep 2019 08:58:52 +0000 Subject: [PATCH 08/16] add protego to install_requires --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 37892cfbf..850456503 100644 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ setup( 'six>=1.10.0', 'w3lib>=1.17.0', 'zope.interface>=4.1.3', + 'protego>=0.1.15', ], extras_require=extras_require, ) From 171fa1cd106f5a4ba34e6d57bdd65c8688564b82 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 10 Sep 2019 09:59:36 +0000 Subject: [PATCH 09/16] documentation rework --- docs/topics/downloader-middleware.rst | 116 +++++++++++++++++--------- 1 file changed, 76 insertions(+), 40 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 192bfd19a..52be8ded2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1000,10 +1000,10 @@ RobotsTxtMiddleware Scrapy ships with support for the following robots.txt_ parsers: - * :ref:`RobotFileParser ` (default) + * :ref:`Protego ` (default) + * :ref:`RobotFileParser ` * :ref:`Reppy ` * :ref:`Robotexclusionrulesparser ` - * :ref:`Protego ` You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` setting. Or you can also :ref:`implement support for a new parser `. @@ -1015,50 +1015,78 @@ If :attr:`Request.meta ` has the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. +Parsers varies in several aspects: + +* Language of implementation + +* Supported specification + +* Support for wildcard matching + +* usage of length based rule: in particular for ``Allow`` and + ``Disallow`` directives, where the most specific rule based on the length of + the path trumps the less specific (shorter) rule + + +.. _protego-parser: + +Protego parser +~~~~~~~~~~~~~~ + +based on `Protego `_: + +* implemented in Python + +* is compliant with `Google's Robots.txt Specification + `_ + +* supports wildcard matching + +* uses the length based rule, + +Scrapy uses this parser by default. + .. _python-robotfileparser: RobotFileParser ~~~~~~~~~~~~~~~ -`RobotFileParser `_ is -Python's built-in robots.txt_ parser. The parser is fully compliant with `Martijn Koster's -1996 draft specification `_. It lacks -support for wildcard matching. +based on `RobotFileParser +`_: + +* is Python's built-in robots.txt_ parser. + +* is compliant with `Martijn Koster's 1996 draft specification + `_. + +* lacks support for wildcard matching. + +* doesn't use the length based rule, + +It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0 . In order to use this parser, set: * :setting:`ROBOTSTXT_PARSER` to ``scrapy.robotstxt.PythonRobotParser`` -.. _rerp-parser: - -Robotexclusionrulesparser -~~~~~~~~~~~~~~~~~~~~~~~~~ - -`Robotexclusionrulesparser `_ is fully compliant -with `Martijn Koster's 1996 draft specification `_, -with support for wildcard matching. - -In order to use this parser: - -* Install `Robotexclusionrulesparser `_ by running - ``pip install robotexclusionrulesparser`` - -* Set :setting:`ROBOTSTXT_PARSER` setting to - ``scrapy.robotstxt.RerpRobotParser`` - .. _reppy-parser: Reppy parser ~~~~~~~~~~~~ -`Reppy `_ is a Python wrapper around `Robots Exclusion -Protocol Parser for C++ `_. The parser is fully compliant -with `Martijn Koster's 1996 draft specification `_, -with support for wildcard matching. Unlike -`RobotFileParser `_ and -`Robotexclusionrulesparser `_, it uses the length based -rule, in particular for ``Allow`` and ``Disallow`` directives, where the most specific -rule based on the length of the path trumps the less specific (shorter) rule. +based on `Reppy `_: + +* is a Python wrapper around `Robots Exclusion Protocol Parser for C++ + `_. + +* is compliant with `Martijn Koster's 1996 draft specification + `_. + +* supports wildcard matching + +* uses the length based rule, + +Native implementation provides better speed than Protego. In order to use this parser: @@ -1067,21 +1095,29 @@ In order to use this parser: * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.ReppyRobotParser`` -.. _protego-parser: +.. _rerp-parser: -Protego parser -~~~~~~~~~~~~~~ +Robotexclusionrulesparser +~~~~~~~~~~~~~~~~~~~~~~~~~ -`Protego `_ is a pure-Python robots.txt_ parser. -The parser is fully compliant with `Google's Robots.txt Specification -`_ hence supports wildcard -matching, and uses the length based rule similar to `Reppy `_. -Scrapy uses this parser by default. +based on `Robotexclusionrulesparser `_: + +* implemented in Python + +* is compliant with `Martijn Koster's 1996 draft specification + `_. + +* supports wildcard matching + +* doesn't use the length based rule, In order to use this parser: +* Install `Robotexclusionrulesparser `_ by running + ``pip install robotexclusionrulesparser`` + * Set :setting:`ROBOTSTXT_PARSER` setting to - ``scrapy.robotstxt.ProtegoRobotParser`` + ``scrapy.robotstxt.RerpRobotParser`` .. _support-for-new-robots-parser: From 66145b4eaf813929c45c17624d3cb4645b2e3ba0 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 12 Sep 2019 18:51:00 +0300 Subject: [PATCH 10/16] Update docs/topics/downloader-middleware.rst Co-Authored-By: Mikhail Korobov --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 52be8ded2..76ee77a35 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1015,7 +1015,7 @@ If :attr:`Request.meta ` has the request will be ignored by this middleware even if :setting:`ROBOTSTXT_OBEY` is enabled. -Parsers varies in several aspects: +Parsers vary in several aspects: * Language of implementation From 5197b39e01226e8153f7a1abb6760b202d2eaf3e Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 16 Sep 2019 14:24:25 +0000 Subject: [PATCH 11/16] fix capitalization, remove commas --- docs/topics/downloader-middleware.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 52be8ded2..f2f754573 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1023,7 +1023,7 @@ Parsers varies in several aspects: * Support for wildcard matching -* usage of length based rule: in particular for ``Allow`` and +* Usage of length based rule: in particular for ``Allow`` and ``Disallow`` directives, where the most specific rule based on the length of the path trumps the less specific (shorter) rule @@ -1033,7 +1033,7 @@ Parsers varies in several aspects: Protego parser ~~~~~~~~~~~~~~ -based on `Protego `_: +Based on `Protego `_: * implemented in Python @@ -1042,7 +1042,7 @@ based on `Protego `_: * supports wildcard matching -* uses the length based rule, +* uses the length based rule Scrapy uses this parser by default. @@ -1051,7 +1051,7 @@ Scrapy uses this parser by default. RobotFileParser ~~~~~~~~~~~~~~~ -based on `RobotFileParser +Based on `RobotFileParser `_: * is Python's built-in robots.txt_ parser. @@ -1061,7 +1061,7 @@ based on `RobotFileParser * lacks support for wildcard matching. -* doesn't use the length based rule, +* doesn't use the length based rule It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0 . @@ -1074,7 +1074,7 @@ In order to use this parser, set: Reppy parser ~~~~~~~~~~~~ -based on `Reppy `_: +Based on `Reppy `_: * is a Python wrapper around `Robots Exclusion Protocol Parser for C++ `_. @@ -1084,7 +1084,7 @@ based on `Reppy `_: * supports wildcard matching -* uses the length based rule, +* uses the length based rule Native implementation provides better speed than Protego. @@ -1100,7 +1100,7 @@ In order to use this parser: Robotexclusionrulesparser ~~~~~~~~~~~~~~~~~~~~~~~~~ -based on `Robotexclusionrulesparser `_: +Based on `Robotexclusionrulesparser `_: * implemented in Python @@ -1109,7 +1109,7 @@ based on `Robotexclusionrulesparser `_: * supports wildcard matching -* doesn't use the length based rule, +* doesn't use the length based rule In order to use this parser: From b792dba5281c91dc6845e3e64771dde2157b20f7 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 17 Sep 2019 06:28:33 +0000 Subject: [PATCH 12/16] remove periods --- docs/topics/downloader-middleware.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b67723b86..de5f72b80 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1054,12 +1054,12 @@ RobotFileParser Based on `RobotFileParser `_: -* is Python's built-in robots.txt_ parser. +* is Python's built-in robots.txt_ parser * is compliant with `Martijn Koster's 1996 draft specification - `_. + `_ -* lacks support for wildcard matching. +* lacks support for wildcard matching * doesn't use the length based rule @@ -1077,10 +1077,10 @@ Reppy parser Based on `Reppy `_: * is a Python wrapper around `Robots Exclusion Protocol Parser for C++ - `_. + `_ * is compliant with `Martijn Koster's 1996 draft specification - `_. + `_ * supports wildcard matching @@ -1105,7 +1105,7 @@ Based on `Robotexclusionrulesparser `_: * implemented in Python * is compliant with `Martijn Koster's 1996 draft specification - `_. + `_ * supports wildcard matching From d39ef77e6be5f1a24858d92018ea4ed7a38bd127 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 17 Sep 2019 06:34:33 +0000 Subject: [PATCH 13/16] add link to google description of lenght-based rule --- docs/topics/downloader-middleware.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index de5f72b80..9ce4293d2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1023,10 +1023,10 @@ Parsers vary in several aspects: * Support for wildcard matching -* Usage of length based rule: in particular for ``Allow`` and - ``Disallow`` directives, where the most specific rule based on the length of - the path trumps the less specific (shorter) rule - +* Usage of `length based rule `_: + in particular for ``Allow`` and ``Disallow`` directives, where the most + specific rule based on the length of the path trumps the less specific + (shorter) rule .. _protego-parser: From 57e6f4c75087b7f2182d2ebc6b28e3afa18e12bd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 17 Sep 2019 07:18:37 +0000 Subject: [PATCH 14/16] add link to performance comparison --- docs/topics/downloader-middleware.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 9ce4293d2..ec302f2eb 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1028,6 +1028,9 @@ Parsers vary in several aspects: specific rule based on the length of the path trumps the less specific (shorter) rule +Performance comparison of different parsers is available at `the following link +`_. + .. _protego-parser: Protego parser From d1d0bf8491da34d1a5a4bcf3d1241241346b62c8 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 17 Sep 2019 12:27:12 +0500 Subject: [PATCH 15/16] Update docs/topics/downloader-middleware.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ec302f2eb..c08e13a9a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1066,7 +1066,7 @@ Based on `RobotFileParser * doesn't use the length based rule -It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0 . +It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. In order to use this parser, set: From 2438ac529a647c6c665d402b59164d509404d584 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 17 Sep 2019 12:27:22 +0500 Subject: [PATCH 16/16] Update docs/topics/downloader-middleware.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/downloader-middleware.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index c08e13a9a..539832618 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1089,7 +1089,7 @@ Based on `Reppy `_: * uses the length based rule -Native implementation provides better speed than Protego. +Native implementation, provides better speed than Protego. In order to use this parser: