Add ability to use FormRequest in contracts

This commit is contained in:
Stas Glubokiy 2018-08-11 22:18:43 +03:00
parent 8a4e51a19b
commit b4b1e48343
3 changed files with 35 additions and 5 deletions

View File

@ -86,8 +86,11 @@ override three methods:
.. method:: Contract.adjust_request_args(args)
This receives a ``dict`` as an argument containing default arguments
for :class:`~scrapy.http.Request` object. Must return the same or a
modified version of it.
for request object. If ``formdata`` is in ``args``, then
:class:`~scrapy.http.FormRequest` object is created,
otherwise :class:`~scrapy.http.Request` is used.
Must return the same or a modified version of it.
.. method:: Contract.pre_process(response)

View File

@ -3,6 +3,7 @@ import re
from functools import wraps
from unittest import TestCase
from scrapy import FormRequest
from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
@ -57,7 +58,11 @@ class ContractsManager(object):
# create and prepare request
args.remove('self')
if set(args).issubset(set(kwargs)):
request = Request(**kwargs)
if 'formdata' in kwargs:
kwargs['method'] = 'POST'
request = FormRequest(**kwargs)
else:
request = Request(**kwargs)
# execute pre and post hooks in order
for contract in reversed(contracts):

View File

@ -3,11 +3,12 @@ from unittest import TextTestResult
from twisted.python import failure
from twisted.trial import unittest
from scrapy import FormRequest
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.item import Item, Field
from scrapy.contracts import ContractsManager
from scrapy.contracts import ContractsManager, Contract
from scrapy.contracts.default import (
UrlContract,
ReturnsContract,
@ -24,6 +25,14 @@ class ResponseMock(object):
url = 'http://scrapy.org'
class CustomFormContract(Contract):
name = 'custom_form'
def adjust_request_args(self, args):
args['formdata'] = {'name': 'scrapy'}
return args
class TestSpider(Spider):
name = 'demo_spider'
@ -100,9 +109,16 @@ class TestSpider(Spider):
"""
pass
def custom_form(self, response):
"""
@url http://scrapy.org
@custom_form
"""
pass
class ContractsManagerTest(unittest.TestCase):
contracts = [UrlContract, ReturnsContract, ScrapesContract]
contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract]
def setUp(self):
self.conman = ContractsManager(self.contracts)
@ -202,3 +218,9 @@ class ContractsManagerTest(unittest.TestCase):
self.assertFalse(self.results.failures)
self.assertTrue(self.results.errors)
def test_form_contract(self):
spider = TestSpider()
request = self.conman.from_method(spider.custom_form, self.results)
self.assertEqual(request.method, 'POST')
self.assertIsInstance(request, FormRequest)