Merge branch 'master' into pylint

docs/conftest.py

@@ -0,0 +1,16 @@
+from doctest import ELLIPSIS
+
+from sybil import Sybil
+from sybil.parsers.codeblock import CodeBlockParser
+from sybil.parsers.doctest import DocTestParser
+from sybil.parsers.skip import skip
+
+
+pytest_collect_file = Sybil(
+    parsers=[
+        DocTestParser(optionflags=ELLIPSIS),
+        CodeBlockParser(future_imports=['print_function']),
+        skip,
+    ],
+    pattern='*.rst',
+).pytest()

docs/topics/jobs.rst

@@ -71,34 +71,11 @@ on cookies.
 Request serialization
 ---------------------
 
-Requests must be serializable by the ``pickle`` module, in order for persistence
-to work, so you should make sure that your requests are serializable.
-
-The most common issue here is to use ``lambda`` functions on request callbacks that
-can't be persisted.
-
-So, for example, this won't work::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com',
-                              callback=lambda r: self.other_callback(r, somearg))
-
-    def other_callback(self, response, somearg):
-        print("the argument passed is: %s" % somearg)
-
-But this will::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com',
-                              callback=self.other_callback, cb_kwargs={'somearg': somearg})
-
-    def other_callback(self, response, somearg):
-        print("the argument passed is: %s" % somearg)
+For persistence to work, :class:`~scrapy.http.Request` objects must be
+serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
+values passed to their ``__init__`` method, which must be methods of the
+runnning :class:`~scrapy.spiders.Spider` class.
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
-
-.. _pickle: https://docs.python.org/library/pickle.html

pytest.ini

@@ -2,7 +2,25 @@
 usefixtures = chdir
 python_files=test_*.py __init__.py
 python_classes=
-addopts = --doctest-modules --assert=plain
+addopts =
+    --assert=plain
+    --doctest-modules
+    --ignore=docs/_ext
+    --ignore=docs/conf.py
+    --ignore=docs/intro/tutorial.rst
+    --ignore=docs/news.rst
+    --ignore=docs/topics/commands.rst
+    --ignore=docs/topics/debug.rst
+    --ignore=docs/topics/developer-tools.rst
+    --ignore=docs/topics/dynamic-content.rst
+    --ignore=docs/topics/items.rst
+    --ignore=docs/topics/leaks.rst
+    --ignore=docs/topics/loaders.rst
+    --ignore=docs/topics/selectors.rst
+    --ignore=docs/topics/shell.rst
+    --ignore=docs/topics/stats.rst
+    --ignore=docs/topics/telnetconsole.rst
+    --ignore=docs/utils
 twisted = 1
 flake8-ignore =
     # extras

tests/requirements-py3.txt

@@ -4,6 +4,7 @@ pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
+sybil
 testfixtures
 
 # optional for shell wrapper tests

tox.ini

@@ -21,7 +21,7 @@ passenv =
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
 commands =
-    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
+    py.test --cov=scrapy --cov-report= {posargs:docs scrapy tests}
 
 [testenv:py35]
 basepython = python3.5
@@ -60,7 +60,7 @@ basepython = python3.8
 [testenv:pypy3]
 basepython = pypy3
 commands =
-    py.test {posargs:scrapy tests}
+    py.test {posargs:docs scrapy tests}
 
 [testenv:flake8]
 basepython = python3.8
@@ -68,7 +68,7 @@ deps =
     {[testenv]deps}
     pytest-flake8
 commands =
-    py.test --flake8 {posargs:scrapy tests}
+    py.test --flake8 {posargs:docs scrapy tests}
     
 [testenv:pylint]
 basepython = python3.8