diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 700238fe9..5e70a4f98 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -911,6 +911,11 @@ Request subclasses Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass it to implement your own custom functionality. +FormRequest +----------- + +.. autoclass:: scrapy.FormRequest + JsonRequest ----------- diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py index e20e894ae..0e5c2b53b 100644 --- a/scrapy/http/__init__.py +++ b/scrapy/http/__init__.py @@ -5,11 +5,9 @@ Use this module (instead of the more specific ones) when importing Headers, Request and Response outside this module. """ -from warnings import catch_warnings, filterwarnings - -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http.headers import Headers from scrapy.http.request import Request +from scrapy.http.request.form import FormRequest from scrapy.http.request.json_request import JsonRequest from scrapy.http.request.rpc import XmlRpcRequest from scrapy.http.response import Response @@ -17,19 +15,6 @@ from scrapy.http.response.html import HtmlResponse from scrapy.http.response.json import JsonResponse from scrapy.http.response.text import TextResponse from scrapy.http.response.xml import XmlResponse -from scrapy.utils.deprecate import create_deprecated_class - -with catch_warnings(): - filterwarnings("ignore", category=ScrapyDeprecationWarning) - - from scrapy.http.request.form import FormRequest as _FormRequest - - FormRequest = create_deprecated_class( - name="FormRequest", - new_class=_FormRequest, - subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.", - instance_warn_message="{cls} is deprecated, use the form2request library instead.", - ) __all__ = [ "FormRequest", diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4da595b22..f1a8dbf3b 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -32,19 +32,61 @@ if TYPE_CHECKING: from scrapy.http.response.text import TextResponse -warn( - "The entire scrapy.http.request.form module is deprecated. Use the " - "form2request library instead.", - ScrapyDeprecationWarning, - stacklevel=2, -) - FormdataVType: TypeAlias = str | Iterable[str] FormdataKVType: TypeAlias = tuple[str, FormdataVType] FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None class FormRequest(Request): + """A :class:`~scrapy.Request` subclass with a ``formdata`` parameter that + url-encodes the given data and assigns it to the request, which makes it + convenient to send arbitrary form data via HTTP POST or GET without an HTML + ``
`` element to parse. + + .. note:: To build a request from an HTML ```` element found in a + response, use :doc:`form2request ` instead. See + :ref:`form`. + + The remaining arguments are the same as for the :class:`~scrapy.Request` + class and are not documented here. + + :param formdata: a dictionary (or iterable of (key, value) tuples) + containing HTML form data which will be url-encoded. If + :attr:`~scrapy.Request.method` is not given and ``formdata`` is + provided, the method is set to ``"POST"`` and the data is assigned to + the request body; if the method is ``"GET"``, the data is added to the + URL query string instead. + :type formdata: dict or collections.abc.Iterable + + To send data via HTTP POST, simulating an HTML form submission, return a + :class:`~scrapy.FormRequest` object from your spider: + + .. skip: next + .. code-block:: python + + return [ + FormRequest( + url="http://www.example.com/post/action", + formdata={"name": "John Doe", "age": "27"}, + callback=self.after_post, + ) + ] + + To send the data in the URL query string instead, use the ``GET`` method: + + .. skip: next + .. code-block:: python + + return [ + FormRequest( + url="http://www.example.com/search", + method="GET", + formdata={"q": "keyword", "page": "1"}, + callback=self.parse_results, + ) + ] + """ + __slots__ = () valid_form_methods: ClassVar[list[str]] = ["GET", "POST"] @@ -84,6 +126,13 @@ class FormRequest(Request): formcss: str | None = None, **kwargs: Any, ) -> Self: + warn( + "FormRequest.from_response() is deprecated. Use the form2request " + "library instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + kwargs.setdefault("encoding", response.encoding) if formcss is not None: diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py index c667e48a1..af86b35c0 100644 --- a/tests/test_http_request_form.py +++ b/tests/test_http_request_form.py @@ -1,10 +1,12 @@ from __future__ import annotations import re +import warnings from urllib.parse import parse_qs, unquote_to_bytes import pytest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import FormRequest, HtmlResponse from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode @@ -26,15 +28,39 @@ def _qs(req, encoding="utf-8", to_unicode=False): return parse_qs(uqs, True) +# FormRequest.from_response() is deprecated in favor of form2request, so the +# many tests below that exercise it ignore the resulting deprecation warning. @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestFormRequest(TestRequest): - request_class = FormRequest # type: ignore[assignment] + request_class = FormRequest def assertQueryEqual(self, first, second, msg=None): first = to_unicode(first).split("&") second = to_unicode(second).split("&") assert sorted(first) == sorted(second), msg + def test_init_not_deprecated(self): + # Building a request directly from form data is not deprecated. + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + self.request_class( + "http://www.example.com", formdata={"a": "1"}, method="POST" + ) + self.request_class( + "http://www.example.com", method="GET", formdata={"a": "1"} + ) + + def test_from_response_deprecated(self): + response = _buildresponse( + """ + + """ + ) + with pytest.warns( + ScrapyDeprecationWarning, match=r"FormRequest\.from_response\(\)" + ): + self.request_class.from_response(response) + def test_empty_formdata(self): r1 = self.request_class("http://www.example.com", formdata={}) assert r1.body == b""