From b4b1e4834376e8565d727a1a5087cae31450f931 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 11 Aug 2018 22:18:43 +0300 Subject: [PATCH 1/2] Add ability to use FormRequest in contracts --- docs/topics/contracts.rst | 7 +++++-- scrapy/contracts/__init__.py | 7 ++++++- tests/test_contracts.py | 26 ++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ba1421c42..cac52042a 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -86,8 +86,11 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for :class:`~scrapy.http.Request` object. Must return the same or a - modified version of it. + for request object. If ``formdata`` is in ``args``, then + :class:`~scrapy.http.FormRequest` object is created, + otherwise :class:`~scrapy.http.Request` is used. + + Must return the same or a modified version of it. .. method:: Contract.pre_process(response) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 8315d21d2..ca2a8d384 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -3,6 +3,7 @@ import re from functools import wraps from unittest import TestCase +from scrapy import FormRequest from scrapy.http import Request from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec @@ -57,7 +58,11 @@ class ContractsManager(object): # create and prepare request args.remove('self') if set(args).issubset(set(kwargs)): - request = Request(**kwargs) + if 'formdata' in kwargs: + kwargs['method'] = 'POST' + request = FormRequest(**kwargs) + else: + request = Request(**kwargs) # execute pre and post hooks in order for contract in reversed(contracts): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b07cbee1e..f2085f711 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -3,11 +3,12 @@ from unittest import TextTestResult from twisted.python import failure from twisted.trial import unittest +from scrapy import FormRequest from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field -from scrapy.contracts import ContractsManager +from scrapy.contracts import ContractsManager, Contract from scrapy.contracts.default import ( UrlContract, ReturnsContract, @@ -24,6 +25,14 @@ class ResponseMock(object): url = 'http://scrapy.org' +class CustomFormContract(Contract): + name = 'custom_form' + + def adjust_request_args(self, args): + args['formdata'] = {'name': 'scrapy'} + return args + + class TestSpider(Spider): name = 'demo_spider' @@ -100,9 +109,16 @@ class TestSpider(Spider): """ pass + def custom_form(self, response): + """ + @url http://scrapy.org + @custom_form + """ + pass + class ContractsManagerTest(unittest.TestCase): - contracts = [UrlContract, ReturnsContract, ScrapesContract] + contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract] def setUp(self): self.conman = ContractsManager(self.contracts) @@ -202,3 +218,9 @@ class ContractsManagerTest(unittest.TestCase): self.assertFalse(self.results.failures) self.assertTrue(self.results.errors) + + def test_form_contract(self): + spider = TestSpider() + request = self.conman.from_method(spider.custom_form, self.results) + self.assertEqual(request.method, 'POST') + self.assertIsInstance(request, FormRequest) From 8dbbbd13950dcb21dda759b073c64ffdca85c2d6 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Mon, 3 Sep 2018 20:07:37 +0300 Subject: [PATCH 2/2] Use request_cls attribute in contract definition --- docs/topics/contracts.rst | 5 +++-- scrapy/contracts/__init__.py | 15 +++++++++------ tests/test_contracts.py | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ada6fd227..70f20d4ed 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -86,8 +86,9 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.http.Request` is used - if ``request_cls`` is not set on ``args``. + for request object. :class:`~scrapy.http.Request` is used by default, + but this can be changed with the ``request_cls`` attribute. + If multiple contracts in chain have this attribute defined, the last one is used. Must return the same or a modified version of it. diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 801c18e73..851a26a8e 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -4,7 +4,6 @@ from functools import wraps from inspect import getmembers from unittest import TestCase -from scrapy import FormRequest from scrapy.http import Request from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec @@ -50,14 +49,17 @@ class ContractsManager(object): def from_method(self, method, results): contracts = self.extract_contracts(method) if contracts: - # prepare request arguments - kwargs = {'callback': method} + request_cls = Request + for contract in contracts: + if contract.request_cls is not None: + request_cls = contract.request_cls + + # calculate request args + args, kwargs = get_spec(request_cls.__init__) + kwargs['callback'] = method for contract in contracts: kwargs = contract.adjust_request_args(kwargs) - request_cls = kwargs.pop('request_cls', Request) - - args, _ = get_spec(request_cls.__init__) args.remove('self') # check if all positional arguments are defined in kwargs @@ -98,6 +100,7 @@ class ContractsManager(object): class Contract(object): """ Abstract class for contracts """ + request_cls = None def __init__(self, method, *args): self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c35b068a4..fc5c94771 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -27,9 +27,9 @@ class ResponseMock(object): class CustomFormContract(Contract): name = 'custom_form' + request_cls = FormRequest def adjust_request_args(self, args): - args['request_cls'] = FormRequest args['formdata'] = {'name': 'scrapy'} return args