Deprecate FormRequest in favor of form2request (#6438)

* WIP

* Add docs

* Remove FormRequest from tests

* Silence mypy issue

* Address docs-tests issues

---------

Co-authored-by: Adrian Chaves <adrian@zyte.com>
Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>

docs/conf.py

@@ -154,6 +154,7 @@ scrapy_intersphinx_enable = [
     "coverage",
     "cryptography",
     "cssselect",
+    "form2request",
     "itemloaders",
     "parsel",
     "pytest",

docs/topics/dynamic-content.rst

@@ -83,7 +83,7 @@ request with Scrapy.
 
 It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
 method and URL. However, you may also need to reproduce the body, headers and
-form parameters (see :class:`~scrapy.FormRequest`) of that request.
+form parameters (see :ref:`form`) of that request.
 
 As all major browsers allow to export the requests in curl_ format, Scrapy
 incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent

docs/topics/leaks.rst

@@ -82,7 +82,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:
     ExampleSpider                       1   oldest: 15s ago
     HtmlResponse                       10   oldest: 1s ago
     Selector                            2   oldest: 0s ago
-    FormRequest                       878   oldest: 7s ago
+    Request                           878   oldest: 7s ago
 
 As you can see, that report also shows the "age" of the oldest object in each
 class. If you're running multiple spiders per process chances are you can

docs/topics/request-response.rst

@@ -246,6 +246,78 @@ Request objects
     .. automethod:: to_dict
 
 
+.. _form:
+
+Creating requests that submit HTML forms
+----------------------------------------
+
+Use :doc:`form2request <form2request:index>` to build request data from an HTML
+``<form>`` element and convert it to a :class:`~scrapy.Request`.
+
+Install it with pip:
+
+.. code-block:: bash
+
+    pip install form2request
+
+Select the desired form with CSS or XPath, then build and convert request
+data:
+
+.. code-block:: python
+
+    from form2request import form2request
+
+
+    def parse(self, response):
+        form = response.css("form#search")
+        request_data = form2request(form, data={"q": "scrapy"})
+        yield request_data.to_scrapy(callback=self.parse_results)
+
+Use ``data`` to override field values. To drop a field from the resulting
+request, set its value to ``None``.
+
+By default, form2request simulates clicking the first submit button. To submit
+without clicking any button, pass ``click=False``. To click a specific submit
+button, pass its element:
+
+.. code-block:: python
+
+    def parse(self, response):
+        form = response.css("form#checkout")
+        submit = form.css('button[name="pay"]')
+        request_data = form2request(form, click=submit)
+
+.. _topics-request-response-ref-request-userlogin:
+
+Using form2request to simulate a user login
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is usual for web sites to provide pre-populated form fields through ``<input
+type="hidden">`` elements, such as session related data or authentication
+tokens (for login pages). Build the request from the form and only override the
+credentials:
+
+.. code-block:: python
+
+    import scrapy
+    from form2request import form2request
+
+
+    class LoginSpider(scrapy.Spider):
+        name = "example.com"
+        start_urls = ["http://www.example.com/users/login.php"]
+
+        def parse(self, response):
+            form = response.css("form")
+            request_data = form2request(
+                form,
+                data={"username": "john", "password": "secret"},
+            )
+            yield request_data.to_scrapy(callback=self.after_login)
+
+        def after_login(self, response): ...
+
+
 Other functions related to requests
 -----------------------------------
 
@@ -771,159 +843,6 @@ Request subclasses
 Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass
 it to implement your own custom functionality.
 
-FormRequest objects
--------------------
-
-The FormRequest class extends the base :class:`~scrapy.Request` with functionality for
-dealing with HTML forms. It uses `lxml.html forms`_  to pre-populate form
-fields with form data from :class:`Response` objects.
-
-.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
-
-.. currentmodule:: None
-
-.. class:: scrapy.FormRequest(url, [formdata, ...])
-    :canonical: scrapy.http.request.form.FormRequest
-
-    The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The
-    remaining arguments are the same as for the :class:`~scrapy.Request` class and are
-    not documented here.
-
-    :param formdata: is a dictionary (or iterable of (key, value) tuples)
-       containing HTML Form data which will be url-encoded and assigned to the
-       body of the request.
-    :type formdata: dict or collections.abc.Iterable
-
-    The :class:`~scrapy.FormRequest` objects support the following class method in
-    addition to the standard :class:`~scrapy.Request` methods:
-
-    .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
-
-       Returns a new :class:`~scrapy.FormRequest` object with its form field values
-       pre-populated with those found in the HTML ``<form>`` element contained
-       in the given response. For an example see
-       :ref:`topics-request-response-ref-request-userlogin`.
-
-       The policy is to automatically simulate a click, by default, on any form
-       control that looks clickable, like a ``<input type="submit">``.  Even
-       though this is quite convenient, and often the desired behaviour,
-       sometimes it can cause problems which could be hard to debug. For
-       example, when working with forms that are filled and/or submitted using
-       javascript, the default :meth:`from_response` behaviour may not be the
-       most appropriate. To disable this behaviour you can set the
-       ``dont_click`` argument to ``True``. Also, if you want to change the
-       control clicked (instead of disabling it) you can also use the
-       ``clickdata`` argument.
-
-       .. caution:: Using this method with select elements which have leading
-          or trailing whitespace in the option values will not work due to a
-          `bug in lxml`_, which should be fixed in lxml 3.8 and above.
-
-       :param response: the response containing a HTML form which will be used
-          to pre-populate the form fields
-       :type response: :class:`~scrapy.http.Response` object
-
-       :param formname: if given, the form with name attribute set to this value will be used.
-       :type formname: str
-
-       :param formid: if given, the form with id attribute set to this value will be used.
-       :type formid: str
-
-       :param formxpath: if given, the first form that matches the xpath will be used.
-       :type formxpath: str
-
-       :param formcss: if given, the first form that matches the css selector will be used.
-       :type formcss: str
-
-       :param formnumber: the number of form to use, when the response contains
-          multiple forms. The first one (and also the default) is ``0``.
-       :type formnumber: int
-
-       :param formdata: fields to override in the form data. If a field was
-          already present in the response ``<form>`` element, its value is
-          overridden by the one passed in this parameter. If a value passed in
-          this parameter is ``None``, the field will not be included in the
-          request, even if it was present in the response ``<form>`` element.
-       :type formdata: dict
-
-       :param clickdata: attributes to lookup the control clicked. If it's not
-         given, the form data will be submitted simulating a click on the
-         first clickable element. In addition to html attributes, the control
-         can be identified by its zero-based index relative to other
-         submittable inputs inside the form, via the ``nr`` attribute.
-       :type clickdata: dict
-
-       :param dont_click: If True, the form data will be submitted without
-         clicking in any element.
-       :type dont_click: bool
-
-       The other parameters of this class method are passed directly to the
-       :class:`~scrapy.FormRequest` ``__init__()`` method.
-
-.. currentmodule:: scrapy.http
-
-Request usage examples
-----------------------
-
-Using FormRequest to send data via HTTP POST
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you want to simulate a HTML Form POST in your spider and send a couple of
-key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your
-spider) like this:
-
-.. skip: next
-.. code-block:: python
-
-   return [
-       FormRequest(
-           url="http://www.example.com/post/action",
-           formdata={"name": "John Doe", "age": "27"},
-           callback=self.after_post,
-       )
-   ]
-
-.. _topics-request-response-ref-request-userlogin:
-
-Using FormRequest.from_response() to simulate a user login
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-It is usual for web sites to provide pre-populated form fields through ``<input
-type="hidden">`` elements, such as session related data or authentication
-tokens (for login pages). When scraping, you'll want these fields to be
-automatically pre-populated and only override a couple of them, such as the
-user name and password. You can use the :meth:`.FormRequest.from_response`
-method for this job. Here's an example spider which uses it:
-
-.. code-block:: python
-
-    import scrapy
-
-
-    def authentication_failed(response):
-        # TODO: Check the contents of the response and return True if it failed
-        # or False if it succeeded.
-        pass
-
-
-    class LoginSpider(scrapy.Spider):
-        name = "example.com"
-        start_urls = ["http://www.example.com/users/login.php"]
-
-        def parse(self, response):
-            return scrapy.FormRequest.from_response(
-                response,
-                formdata={"username": "john", "password": "secret"},
-                callback=self.after_login,
-            )
-
-        def after_login(self, response):
-            if authentication_failed(response):
-                self.logger.error("Login failed")
-                return
-
-            # continue scraping with authenticated session...
-
 JsonRequest
 -----------
 

scrapy/http/__init__.py

@@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers,
 Request and Response outside this module.
 """
 
+from warnings import catch_warnings, filterwarnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http.headers import Headers
 from scrapy.http.request import Request
-from scrapy.http.request.form import FormRequest
 from scrapy.http.request.json_request import JsonRequest
 from scrapy.http.request.rpc import XmlRpcRequest
 from scrapy.http.response import Response
@@ -15,6 +17,19 @@ from scrapy.http.response.html import HtmlResponse
 from scrapy.http.response.json import JsonResponse
 from scrapy.http.response.text import TextResponse
 from scrapy.http.response.xml import XmlResponse
+from scrapy.utils.deprecate import create_deprecated_class
+
+with catch_warnings():
+    filterwarnings("ignore", category=ScrapyDeprecationWarning)
+
+    from scrapy.http.request.form import FormRequest as _FormRequest
+
+    FormRequest = create_deprecated_class(
+        name="FormRequest",
+        new_class=_FormRequest,
+        subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",
+        instance_warn_message="{cls} is deprecated, use the form2request library instead.",
+    )
 
 __all__ = [
     "FormRequest",

scrapy/http/request/form.py

@@ -10,10 +10,12 @@ from __future__ import annotations
 from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
+from warnings import warn
 
 from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http.request import Request
 from scrapy.utils.python import is_listlike, to_bytes
 
@@ -30,6 +32,12 @@ if TYPE_CHECKING:
 
     from scrapy.http.response.text import TextResponse
 
+warn(
+    "The entire scrapy.http.request.form module is deprecated. Use the "
+    "form2request library instead.",
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)
 
 FormdataVType: TypeAlias = str | Iterable[str]
 FormdataKVType: TypeAlias = tuple[str, FormdataVType]

tests/test_contracts.py

@@ -3,7 +3,6 @@ from unittest import TextTestResult
 import pytest
 from twisted.python import failure
 
-from scrapy import FormRequest
 from scrapy.contracts import Contract, ContractsManager
 from scrapy.contracts.default import (
     CallbackKeywordArgumentsContract,
@@ -34,6 +33,12 @@ class ResponseMetaMock(ResponseMock):
     meta = None
 
 
+class TaggedRequest(Request):
+    def __init__(self, url, contract_tag=None, **kwargs):
+        super().__init__(url, **kwargs)
+        self.contract_tag = contract_tag
+
+
 class CustomSuccessContract(Contract):
     name = "custom_success_contract"
 
@@ -49,12 +54,13 @@ class CustomFailContract(Contract):
         raise TypeError("Error in adjust_request_args")
 
 
-class CustomFormContract(Contract):
-    name = "custom_form"
-    request_cls = FormRequest
+class CustomTaggedRequestContract(Contract):
+    name = "custom_tagged_request"
+    request_cls = TaggedRequest
 
     def adjust_request_args(self, args):
-        args["formdata"] = {"name": "scrapy"}
+        args["contract_tag"] = "custom"
+        args["method"] = "POST"
         return args
 
 
@@ -179,10 +185,10 @@ class DemoSpider(Spider):
         @returns items 1 1
         """
 
-    def custom_form(self, response):
+    def custom_tagged_request(self, response):
         """
         @url http://scrapy.org
-        @custom_form
+        @custom_tagged_request
         """
 
     def invalid_regex(self, response):
@@ -253,7 +259,7 @@ class TestContractsManager:
         MetadataContract,
         ReturnsContract,
         ScrapesContract,
-        CustomFormContract,
+        CustomTaggedRequestContract,
         CustomSuccessContract,
         CustomFailContract,
     ]
@@ -533,17 +539,21 @@ class TestContractsManager:
 
         assert crawler.spider.visited == 2
 
-    def test_form_contract(self):
+    def test_custom_tagged_request_contract(self):
         spider = DemoSpider()
-        request = self.conman.from_method(spider.custom_form, self.results)
+        request = self.conman.from_method(spider.custom_tagged_request, self.results)
         assert request.method == "POST"
-        assert isinstance(request, FormRequest)
+        assert isinstance(request, TaggedRequest)
+        assert request.contract_tag == "custom"
 
     def test_inherited_contracts(self):
         spider = InheritsDemoSpider()
 
         requests = self.conman.from_spider(spider, self.results)
         assert requests
+        assert any(
+            isinstance(request, TaggedRequest) for request in requests if request
+        )
 
 
 class CustomFailContractPreProcess(Contract):

tests/test_http_request_form.py

@@ -27,7 +27,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):
 
 
 class TestFormRequest(TestRequest):
-    request_class = FormRequest
+    request_class = FormRequest  # type: ignore[assignment]
 
     def assertQueryEqual(self, first, second, msg=None):
         first = to_unicode(first).split("&")

tests/test_request_dict.py

@@ -1,7 +1,7 @@
 import pytest
 
 from scrapy import Request, Spider
-from scrapy.http import FormRequest, JsonRequest
+from scrapy.http import JsonRequest
 from scrapy.utils.request import request_from_dict
 
 
@@ -67,12 +67,10 @@ class TestRequestSerialization:
             assert r1.dumps_kwargs == r2.dumps_kwargs
 
     def test_request_class(self):
-        r1 = FormRequest("http://www.example.com")
+        r1 = CustomRequest("http://www.example.com")
         self._assert_serializes_ok(r1, spider=self.spider)
-        r2 = CustomRequest("http://www.example.com")
+        r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
         self._assert_serializes_ok(r2, spider=self.spider)
-        r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
-        self._assert_serializes_ok(r3, spider=self.spider)
 
     def test_callback_serialization(self):
         r = Request(

tox.ini

@@ -245,6 +245,7 @@ changedir = docs
 deps =
     {[test-requirements]deps}
     -rdocs/requirements.txt
+    form2request
 commands =
     pytest