From aef98188facfc79dc574d8a86200b4e95b96b880 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 7 Nov 2019 18:06:55 +0100
Subject: [PATCH 1/2] Improve the details about request serialization
 requirements for JOBDIR

---
 docs/topics/jobs.rst | 31 ++++---------------------------
 1 file changed, 4 insertions(+), 27 deletions(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 9fd311c69..f5542495b 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -71,34 +71,11 @@ on cookies.
 Request serialization
 ---------------------
 
-Requests must be serializable by the ``pickle`` module, in order for persistence
-to work, so you should make sure that your requests are serializable.
-
-The most common issue here is to use ``lambda`` functions on request callbacks that
-can't be persisted.
-
-So, for example, this won't work::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com',
-                              callback=lambda r: self.other_callback(r, somearg))
-
-    def other_callback(self, response, somearg):
-        print("the argument passed is: %s" % somearg)
-
-But this will::
-
-    def some_callback(self, response):
-        somearg = 'test'
-        return scrapy.Request('http://www.example.com',
-                              callback=self.other_callback, cb_kwargs={'somearg': somearg})
-
-    def other_callback(self, response, somearg):
-        print("the argument passed is: %s" % somearg)
+For persistence to work, :class:`~scrapy.http.Request` objects must be
+serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
+values passed to their ``__init__`` method, which must be methods of the
+runnning :class:`~scrapy.spiders.Spider` class.
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
-
-.. _pickle: https://docs.python.org/library/pickle.html

From 1df5755699eac5a98239ae73dfb82908706bf03b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 8 Nov 2019 16:00:10 +0100
Subject: [PATCH 2/2] Set the bases for testing code examples from the
 documentation

---
 docs/conftest.py           | 16 ++++++++++++++++
 pytest.ini                 | 20 +++++++++++++++++++-
 tests/requirements-py3.txt |  1 +
 tox.ini                    |  6 +++---
 4 files changed, 39 insertions(+), 4 deletions(-)
 create mode 100644 docs/conftest.py

diff --git a/docs/conftest.py b/docs/conftest.py
new file mode 100644
index 000000000..91c1d4428
--- /dev/null
+++ b/docs/conftest.py
@@ -0,0 +1,16 @@
+from doctest import ELLIPSIS
+
+from sybil import Sybil
+from sybil.parsers.codeblock import CodeBlockParser
+from sybil.parsers.doctest import DocTestParser
+from sybil.parsers.skip import skip
+
+
+pytest_collect_file = Sybil(
+    parsers=[
+        DocTestParser(optionflags=ELLIPSIS),
+        CodeBlockParser(future_imports=['print_function']),
+        skip,
+    ],
+    pattern='*.rst',
+).pytest()
diff --git a/pytest.ini b/pytest.ini
index db5bee228..8c5a2cd54 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -2,7 +2,25 @@
 usefixtures = chdir
 python_files=test_*.py __init__.py
 python_classes=
-addopts = --doctest-modules --assert=plain
+addopts =
+    --assert=plain
+    --doctest-modules
+    --ignore=docs/_ext
+    --ignore=docs/conf.py
+    --ignore=docs/intro/tutorial.rst
+    --ignore=docs/news.rst
+    --ignore=docs/topics/commands.rst
+    --ignore=docs/topics/debug.rst
+    --ignore=docs/topics/developer-tools.rst
+    --ignore=docs/topics/dynamic-content.rst
+    --ignore=docs/topics/items.rst
+    --ignore=docs/topics/leaks.rst
+    --ignore=docs/topics/loaders.rst
+    --ignore=docs/topics/selectors.rst
+    --ignore=docs/topics/shell.rst
+    --ignore=docs/topics/stats.rst
+    --ignore=docs/topics/telnetconsole.rst
+    --ignore=docs/utils
 twisted = 1
 flake8-ignore =
     # extras
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index dd5b23cc3..2e8d319d2 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -4,6 +4,7 @@ pytest
 pytest-cov
 pytest-twisted
 pytest-xdist
+sybil
 testfixtures
 
 # optional for shell wrapper tests
diff --git a/tox.ini b/tox.ini
index cc3463a4d..3668058c3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,7 +21,7 @@ passenv =
     GCS_TEST_FILE_URI
     GCS_PROJECT_ID
 commands =
-    py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
+    py.test --cov=scrapy --cov-report= {posargs:docs scrapy tests}
 
 [testenv:py35]
 basepython = python3.5
@@ -60,7 +60,7 @@ basepython = python3.8
 [testenv:pypy3]
 basepython = pypy3
 commands =
-    py.test {posargs:scrapy tests}
+    py.test {posargs:docs scrapy tests}
 
 [testenv:flake8]
 basepython = python3.8
@@ -68,7 +68,7 @@ deps =
     {[testenv]deps}
     pytest-flake8
 commands =
-    py.test --flake8 {posargs:scrapy tests}
+    py.test --flake8 {posargs:docs scrapy tests}
 
 [docs]
 changedir = docs