From 8d69a7c8659ff207420a8c5d725fe0cafa2b866c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 4 May 2026 19:38:29 +0200 Subject: [PATCH] Deprecate FormRequest in favor of form2request (#6438) * WIP * Add docs * Remove FormRequest from tests * Silence mypy issue * Address docs-tests issues --------- Co-authored-by: Adrian Chaves Co-authored-by: Andrey Rakhmatullin --- docs/conf.py | 1 + docs/topics/dynamic-content.rst | 2 +- docs/topics/leaks.rst | 2 +- docs/topics/request-response.rst | 225 ++++++++++--------------------- scrapy/http/__init__.py | 17 ++- scrapy/http/request/form.py | 8 ++ tests/test_contracts.py | 32 +++-- tests/test_http_request_form.py | 2 +- tests/test_request_dict.py | 8 +- tox.ini | 1 + 10 files changed, 125 insertions(+), 173 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e3e618860..99d5df7da 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -154,6 +154,7 @@ scrapy_intersphinx_enable = [ "coverage", "cryptography", "cssselect", + "form2request", "itemloaders", "parsel", "pytest", diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 090caa6e0..5a399e094 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -83,7 +83,7 @@ request with Scrapy. It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.FormRequest`) of that request. +form parameters (see :ref:`form`) of that request. As all major browsers allow to export the requests in curl_ format, Scrapy incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index ff05cd284..0913a3310 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -82,7 +82,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function: ExampleSpider 1 oldest: 15s ago HtmlResponse 10 oldest: 1s ago Selector 2 oldest: 0s ago - FormRequest 878 oldest: 7s ago + Request 878 oldest: 7s ago As you can see, that report also shows the "age" of the oldest object in each class. If you're running multiple spiders per process chances are you can diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f23c7611c..ba7b3ee3c 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -246,6 +246,78 @@ Request objects .. automethod:: to_dict +.. _form: + +Creating requests that submit HTML forms +---------------------------------------- + +Use :doc:`form2request ` to build request data from an HTML +``
`` element and convert it to a :class:`~scrapy.Request`. + +Install it with pip: + +.. code-block:: bash + + pip install form2request + +Select the desired form with CSS or XPath, then build and convert request +data: + +.. code-block:: python + + from form2request import form2request + + + def parse(self, response): + form = response.css("form#search") + request_data = form2request(form, data={"q": "scrapy"}) + yield request_data.to_scrapy(callback=self.parse_results) + +Use ``data`` to override field values. To drop a field from the resulting +request, set its value to ``None``. + +By default, form2request simulates clicking the first submit button. To submit +without clicking any button, pass ``click=False``. To click a specific submit +button, pass its element: + +.. code-block:: python + + def parse(self, response): + form = response.css("form#checkout") + submit = form.css('button[name="pay"]') + request_data = form2request(form, click=submit) + +.. _topics-request-response-ref-request-userlogin: + +Using form2request to simulate a user login +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is usual for web sites to provide pre-populated form fields through ```` elements, such as session related data or authentication +tokens (for login pages). Build the request from the form and only override the +credentials: + +.. code-block:: python + + import scrapy + from form2request import form2request + + + class LoginSpider(scrapy.Spider): + name = "example.com" + start_urls = ["http://www.example.com/users/login.php"] + + def parse(self, response): + form = response.css("form") + request_data = form2request( + form, + data={"username": "john", "password": "secret"}, + ) + yield request_data.to_scrapy(callback=self.after_login) + + def after_login(self, response): ... + + Other functions related to requests ----------------------------------- @@ -771,159 +843,6 @@ Request subclasses Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass it to implement your own custom functionality. -FormRequest objects -------------------- - -The FormRequest class extends the base :class:`~scrapy.Request` with functionality for -dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form -fields with form data from :class:`Response` objects. - -.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms - -.. currentmodule:: None - -.. class:: scrapy.FormRequest(url, [formdata, ...]) - :canonical: scrapy.http.request.form.FormRequest - - The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The - remaining arguments are the same as for the :class:`~scrapy.Request` class and are - not documented here. - - :param formdata: is a dictionary (or iterable of (key, value) tuples) - containing HTML Form data which will be url-encoded and assigned to the - body of the request. - :type formdata: dict or collections.abc.Iterable - - The :class:`~scrapy.FormRequest` objects support the following class method in - addition to the standard :class:`~scrapy.Request` methods: - - .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...]) - - Returns a new :class:`~scrapy.FormRequest` object with its form field values - pre-populated with those found in the HTML ```` element contained - in the given response. For an example see - :ref:`topics-request-response-ref-request-userlogin`. - - The policy is to automatically simulate a click, by default, on any form - control that looks clickable, like a ````. Even - though this is quite convenient, and often the desired behaviour, - sometimes it can cause problems which could be hard to debug. For - example, when working with forms that are filled and/or submitted using - javascript, the default :meth:`from_response` behaviour may not be the - most appropriate. To disable this behaviour you can set the - ``dont_click`` argument to ``True``. Also, if you want to change the - control clicked (instead of disabling it) you can also use the - ``clickdata`` argument. - - .. caution:: Using this method with select elements which have leading - or trailing whitespace in the option values will not work due to a - `bug in lxml`_, which should be fixed in lxml 3.8 and above. - - :param response: the response containing a HTML form which will be used - to pre-populate the form fields - :type response: :class:`~scrapy.http.Response` object - - :param formname: if given, the form with name attribute set to this value will be used. - :type formname: str - - :param formid: if given, the form with id attribute set to this value will be used. - :type formid: str - - :param formxpath: if given, the first form that matches the xpath will be used. - :type formxpath: str - - :param formcss: if given, the first form that matches the css selector will be used. - :type formcss: str - - :param formnumber: the number of form to use, when the response contains - multiple forms. The first one (and also the default) is ``0``. - :type formnumber: int - - :param formdata: fields to override in the form data. If a field was - already present in the response ```` element, its value is - overridden by the one passed in this parameter. If a value passed in - this parameter is ``None``, the field will not be included in the - request, even if it was present in the response ```` element. - :type formdata: dict - - :param clickdata: attributes to lookup the control clicked. If it's not - given, the form data will be submitted simulating a click on the - first clickable element. In addition to html attributes, the control - can be identified by its zero-based index relative to other - submittable inputs inside the form, via the ``nr`` attribute. - :type clickdata: dict - - :param dont_click: If True, the form data will be submitted without - clicking in any element. - :type dont_click: bool - - The other parameters of this class method are passed directly to the - :class:`~scrapy.FormRequest` ``__init__()`` method. - -.. currentmodule:: scrapy.http - -Request usage examples ----------------------- - -Using FormRequest to send data via HTTP POST -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to simulate a HTML Form POST in your spider and send a couple of -key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your -spider) like this: - -.. skip: next -.. code-block:: python - - return [ - FormRequest( - url="http://www.example.com/post/action", - formdata={"name": "John Doe", "age": "27"}, - callback=self.after_post, - ) - ] - -.. _topics-request-response-ref-request-userlogin: - -Using FormRequest.from_response() to simulate a user login -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is usual for web sites to provide pre-populated form fields through ```` elements, such as session related data or authentication -tokens (for login pages). When scraping, you'll want these fields to be -automatically pre-populated and only override a couple of them, such as the -user name and password. You can use the :meth:`.FormRequest.from_response` -method for this job. Here's an example spider which uses it: - -.. code-block:: python - - import scrapy - - - def authentication_failed(response): - # TODO: Check the contents of the response and return True if it failed - # or False if it succeeded. - pass - - - class LoginSpider(scrapy.Spider): - name = "example.com" - start_urls = ["http://www.example.com/users/login.php"] - - def parse(self, response): - return scrapy.FormRequest.from_response( - response, - formdata={"username": "john", "password": "secret"}, - callback=self.after_login, - ) - - def after_login(self, response): - if authentication_failed(response): - self.logger.error("Login failed") - return - - # continue scraping with authenticated session... - JsonRequest ----------- diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index 0e5c2b53b..e20e894ae 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers, Request and Response outside this module. """ +from warnings import catch_warnings, filterwarnings + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.headers import Headers from scrapy.http.request import Request -from scrapy.http.request.form import FormRequest from scrapy.http.request.json_request import JsonRequest from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response @@ -15,6 +17,19 @@ from scrapy.http.response.html import HtmlResponse from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse +from scrapy.utils.deprecate import create_deprecated_class + +with catch_warnings(): + filterwarnings("ignore", category=ScrapyDeprecationWarning) + + from scrapy.http.request.form import FormRequest as _FormRequest + + FormRequest = create_deprecated_class( + name="FormRequest", + new_class=_FormRequest, + subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.", + instance_warn_message="{cls} is deprecated, use the form2request library instead.", + ) __all__ = [ "FormRequest", diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index affa14499..4da595b22 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -10,10 +10,12 @@ from __future__ import annotations from collections.abc import Iterable from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit +from warnings import warn from parsel.csstranslator import HTMLTranslator from w3lib.html import strip_html5_whitespace +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.request import Request from scrapy.utils.python import is_listlike, to_bytes @@ -30,6 +32,12 @@ if TYPE_CHECKING: from scrapy.http.response.text import TextResponse +warn( + "The entire scrapy.http.request.form module is deprecated. Use the " + "form2request library instead.", + ScrapyDeprecationWarning, + stacklevel=2, +) FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] diff --git a/tests/test_contracts.py b/tests/test_contracts.py index a35ef7010..008e326ec 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -3,7 +3,6 @@ from unittest import TextTestResult import pytest from twisted.python import failure -from scrapy import FormRequest from scrapy.contracts import Contract, ContractsManager from scrapy.contracts.default import ( CallbackKeywordArgumentsContract, @@ -34,6 +33,12 @@ class ResponseMetaMock(ResponseMock): meta = None +class TaggedRequest(Request): + def __init__(self, url, contract_tag=None, **kwargs): + super().__init__(url, **kwargs) + self.contract_tag = contract_tag + + class CustomSuccessContract(Contract): name = "custom_success_contract" @@ -49,12 +54,13 @@ class CustomFailContract(Contract): raise TypeError("Error in adjust_request_args") -class CustomFormContract(Contract): - name = "custom_form" - request_cls = FormRequest +class CustomTaggedRequestContract(Contract): + name = "custom_tagged_request" + request_cls = TaggedRequest def adjust_request_args(self, args): - args["formdata"] = {"name": "scrapy"} + args["contract_tag"] = "custom" + args["method"] = "POST" return args @@ -179,10 +185,10 @@ class DemoSpider(Spider): @returns items 1 1 """ - def custom_form(self, response): + def custom_tagged_request(self, response): """ @url http://scrapy.org - @custom_form + @custom_tagged_request """ def invalid_regex(self, response): @@ -253,7 +259,7 @@ class TestContractsManager: MetadataContract, ReturnsContract, ScrapesContract, - CustomFormContract, + CustomTaggedRequestContract, CustomSuccessContract, CustomFailContract, ] @@ -533,17 +539,21 @@ class TestContractsManager: assert crawler.spider.visited == 2 - def test_form_contract(self): + def test_custom_tagged_request_contract(self): spider = DemoSpider() - request = self.conman.from_method(spider.custom_form, self.results) + request = self.conman.from_method(spider.custom_tagged_request, self.results) assert request.method == "POST" - assert isinstance(request, FormRequest) + assert isinstance(request, TaggedRequest) + assert request.contract_tag == "custom" def test_inherited_contracts(self): spider = InheritsDemoSpider() requests = self.conman.from_spider(spider, self.results) assert requests + assert any( + isinstance(request, TaggedRequest) for request in requests if request + ) class CustomFailContractPreProcess(Contract): diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index 92f2f3fac..8ba9f3422 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -27,7 +27,7 @@ def _qs(req, encoding="utf-8", to_unicode=False): class TestFormRequest(TestRequest): - request_class = FormRequest + request_class = FormRequest # type: ignore[assignment] def assertQueryEqual(self, first, second, msg=None): first = to_unicode(first).split("&") diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py index ea7018541..78ff18b15 100644 --- a/tests/test_request_dict.py +++ b/tests/test_request_dict.py @@ -1,7 +1,7 @@ import pytest from scrapy import Request, Spider -from scrapy.http import FormRequest, JsonRequest +from scrapy.http import JsonRequest from scrapy.utils.request import request_from_dict @@ -67,12 +67,10 @@ class TestRequestSerialization: assert r1.dumps_kwargs == r2.dumps_kwargs def test_request_class(self): - r1 = FormRequest("http://www.example.com") + r1 = CustomRequest("http://www.example.com") self._assert_serializes_ok(r1, spider=self.spider) - r2 = CustomRequest("http://www.example.com") + r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) self._assert_serializes_ok(r2, spider=self.spider) - r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4}) - self._assert_serializes_ok(r3, spider=self.spider) def test_callback_serialization(self): r = Request( diff --git a/tox.ini b/tox.ini index 219a2b616..f1b004ab8 100644 --- a/tox.ini +++ b/tox.ini @@ -245,6 +245,7 @@ changedir = docs deps = {[test-requirements]deps} -rdocs/requirements.txt + form2request commands = pytest