diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 61aef4bbb..df67bee02 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -30,43 +30,15 @@ You can use the following contracts: .. module:: scrapy.contracts.default -.. class:: UrlContract +.. autoclass:: UrlContract - This contract (``@url``) sets the sample URL used when checking other - contract conditions for this spider. This contract is mandatory. All - callbacks lacking this contract are ignored when running the checks:: +.. autoclass:: CallbackKeywordArgumentsContract - @url url +.. autoclass:: MetadataContract -.. class:: CallbackKeywordArgumentsContract +.. autoclass:: ReturnsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} - -.. class:: MetadataContract - - This contract (``@meta``) sets the :attr:`meta ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @meta {"arg1": "value1", "arg2": "value2", ...} - -.. class:: ReturnsContract - - This contract (``@returns``) sets lower and upper bounds for the items and - requests returned by the spider. The upper bound is optional:: - - @returns item(s)|request(s) [min [max]] - -.. class:: ScrapesContract - - This contract (``@scrapes``) checks that all the items returned by the - callback have the specified fields:: - - @scrapes field_1 field_2 ... +.. autoclass:: ScrapesContract Use the :command:`check` command to run the contract checks. @@ -89,30 +61,16 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, *args) +.. autoclass:: Contract - :param method: callback function to which the contract is associated - :type method: collections.abc.Callable + .. automethod:: adjust_request_args - :param args: list of arguments passed into the docstring (whitespace - separated) - :type args: list - - .. method:: Contract.adjust_request_args(args) - - This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.Request` is used by default, - but this can be changed with the ``request_cls`` attribute. - If multiple contracts in chain have this attribute defined, the last one is used. - - Must return the same or a modified version of it. - - .. method:: Contract.pre_process(response) + .. method:: pre_process(response) This allows hooking in various checks on the response received from the sample request, before it's being passed to the callback. - .. method:: Contract.post_process(output) + .. method:: post_process(output) This allows processing the output of the callback. Iterators are converted to lists before being passed to this hook. diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index ebbdf1b98..dbdbbd456 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -22,7 +22,16 @@ if TYPE_CHECKING: class Contract: - """Abstract class for contracts""" + """Base class for :ref:`custom contracts `. + + *method* is the callback function to which the contract is associated. + + *args* is the list of arguments passed into the docstring, separated by + whitespace. + + Subclasses may override :meth:`adjust_request_args`, and define a + ``pre_process`` method or a ``post_process`` method, or both. + """ request_cls: type[Request] | None = None name: str @@ -90,6 +99,13 @@ class Contract: return request def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]: + """Receive a ``dict`` with the default arguments for the sample request + and return it, either unmodified or with changes. + + :class:`~scrapy.Request` is used by default, but this can be changed + with the ``request_cls`` attribute. If multiple contracts in the chain + define this attribute, the last one is used. + """ return args diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 9b42ca36f..e2e27165a 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -15,8 +15,15 @@ if TYPE_CHECKING: # contracts class UrlContract(Contract): - """Contract to set the url of the request (mandatory) - @url http://scrapy.org + """Sets (``@url``) the sample URL used when checking the other contract + conditions of a callback. + + This contract is mandatory: callbacks lacking it are ignored when running + the checks. + + .. code-block:: none + + @url url """ name = "url" @@ -27,10 +34,14 @@ class UrlContract(Contract): class CallbackKeywordArgumentsContract(Contract): - """Contract to set the keyword arguments for the request. - The value should be a JSON-encoded dictionary, e.g.: + """Sets (``@cb_kwargs``) the :attr:`cb_kwargs ` + attribute of the sample request. - @cb_kwargs {"arg1": "some value"} + Its value must be a valid JSON dictionary. + + .. code-block:: none + + @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} """ name = "cb_kwargs" @@ -41,10 +52,14 @@ class CallbackKeywordArgumentsContract(Contract): class MetadataContract(Contract): - """Contract to set metadata arguments for the request. - The value should be JSON-encoded dictionary, e.g.: + """Sets (``@meta``) the :attr:`meta ` attribute of the + sample request. - @meta {"arg1": "some value"} + Its value must be a valid JSON dictionary. + + .. code-block:: none + + @meta {"arg1": "value1", "arg2": "value2", ...} """ name = "meta" @@ -55,16 +70,29 @@ class MetadataContract(Contract): class ReturnsContract(Contract): - """Contract to check the output of a callback + """Sets (``@returns``) lower and upper bounds for the items and requests + returned by a callback. - general form: - @returns request(s)/item(s) [min=1 [max]] + The upper bound is optional: - e.g.: - @returns request - @returns request 2 - @returns request 2 10 - @returns request 0 10 + .. code-block:: none + + @returns item(s)|request(s) [min [max]] + + For example: + + .. code-block:: none + + @returns request + @returns request 2 + @returns request 2 10 + @returns request 0 10 + + Set both bounds to the same value to require an exact number: + + .. code-block:: none + + @returns request 2 2 """ name = "returns" @@ -115,8 +143,12 @@ class ReturnsContract(Contract): class ScrapesContract(Contract): - """Contract to check presence of fields in scraped items - @scrapes page_name page_body + """Checks (``@scrapes``) that all items returned by a callback have the + specified fields. + + .. code-block:: none + + @scrapes field_1 field_2 ... """ name = "scrapes"