From 8fc017d345af1e6fa53334fc9df92ebd8f9dc32e Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 11 Aug 2018 19:25:33 +0300 Subject: [PATCH 1/6] Add dont_filter to ContractsManager requests --- scrapy/contracts/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 8315d21d2..2569ab151 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -50,7 +50,12 @@ class ContractsManager(object): if contracts: # calculate request args args, kwargs = get_spec(Request.__init__) + + # Don't filter requests to allow + # testing different callbacks on the same URL. + kwargs['dont_filter'] = True kwargs['callback'] = method + for contract in contracts: kwargs = contract.adjust_request_args(kwargs) From 4de493efdd80d8ff85a78009dcc120ac46b9c55c Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Wed, 15 Aug 2018 20:24:00 +0300 Subject: [PATCH 2/6] Add test_same_url --- tests/test_contracts.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b07cbee1e..e4f1a777b 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,8 +1,10 @@ from unittest import TextTestResult +from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest +from scrapy.crawler import CrawlerRunner from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request @@ -101,6 +103,29 @@ class TestSpider(Spider): pass +class TestSameUrlSpider(Spider): + + name = 'test_same_url' + + def __init__(self, *args, **kwargs): + super(TestSameUrlSpider, self).__init__(*args, **kwargs) + self.visited = 0 + + def parse_first(self, response): + """first callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + def parse_second(self, response): + """second callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + class ContractsManagerTest(unittest.TestCase): contracts = [UrlContract, ReturnsContract, ScrapesContract] @@ -177,14 +202,12 @@ class ContractsManagerTest(unittest.TestCase): self.should_succeed() # scrapes_item_fail - request = self.conman.from_method(spider.scrapes_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_item_fail, self.results) request.callback(response) self.should_fail() # scrapes_dict_item_fail - request = self.conman.from_method(spider.scrapes_dict_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results) request.callback(response) self.should_fail() @@ -202,3 +225,12 @@ class ContractsManagerTest(unittest.TestCase): self.assertFalse(self.results.failures) self.assertTrue(self.results.errors) + + @defer.inlineCallbacks + def test_same_url(self): + TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results) + + crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + yield crawler.crawl() + + self.assertEqual(crawler.spider.visited, 2) From 2cb4decb6ad52548b3441c877235c540cef12082 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Wed, 15 Aug 2018 20:36:10 +0300 Subject: [PATCH 3/6] Move TestSameUrlSpider to test method --- tests/test_contracts.py | 45 ++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index e4f1a777b..430d89253 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -103,29 +103,6 @@ class TestSpider(Spider): pass -class TestSameUrlSpider(Spider): - - name = 'test_same_url' - - def __init__(self, *args, **kwargs): - super(TestSameUrlSpider, self).__init__(*args, **kwargs) - self.visited = 0 - - def parse_first(self, response): - """first callback - @url http://scrapy.org - """ - self.visited += 1 - return TestItem() - - def parse_second(self, response): - """second callback - @url http://scrapy.org - """ - self.visited += 1 - return TestItem() - - class ContractsManagerTest(unittest.TestCase): contracts = [UrlContract, ReturnsContract, ScrapesContract] @@ -228,6 +205,28 @@ class ContractsManagerTest(unittest.TestCase): @defer.inlineCallbacks def test_same_url(self): + + class TestSameUrlSpider(Spider): + name = 'test_same_url' + + def __init__(self, *args, **kwargs): + super(TestSameUrlSpider, self).__init__(*args, **kwargs) + self.visited = 0 + + def parse_first(self, response): + """first callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + def parse_second(self, response): + """second callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results) crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) From ddd69f4c10578975658ef6ac450cbe1bed85df80 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Wed, 15 Aug 2018 20:39:43 +0300 Subject: [PATCH 4/6] Use MockServer in test_same_url --- tests/test_contracts.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 430d89253..223a926f1 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -15,6 +15,7 @@ from scrapy.contracts.default import ( ReturnsContract, ScrapesContract, ) +from tests.mockserver import MockServer class TestItem(Item): @@ -213,6 +214,9 @@ class ContractsManagerTest(unittest.TestCase): super(TestSameUrlSpider, self).__init__(*args, **kwargs) self.visited = 0 + def start_requests(s): + return self.conman.from_spider(s, self.results) + def parse_first(self, response): """first callback @url http://scrapy.org @@ -227,9 +231,8 @@ class ContractsManagerTest(unittest.TestCase): self.visited += 1 return TestItem() - TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results) - crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) - yield crawler.crawl() + with MockServer() as mockserver: + yield crawler.crawl(mockserver=mockserver) self.assertEqual(crawler.spider.visited, 2) From 0467737cf0bc4c5603bd63a87e220482eb724600 Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sat, 18 Aug 2018 15:42:21 +0300 Subject: [PATCH 5/6] Fix mockserver usage --- tests/test_contracts.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index c5820f256..0a216b745 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -222,22 +222,20 @@ class ContractsManagerTest(unittest.TestCase): return self.conman.from_spider(s, self.results) def parse_first(self, response): - """first callback - @url {} - """.format(self.mockserver.url('/status?n=200')) self.visited += 1 return TestItem() def parse_second(self, response): - """second callback - @url {} - """.format(self.mockserver.url('/status?n=200')) self.visited += 1 return TestItem() - crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) with MockServer() as mockserver: - yield crawler.crawl(mockserver=mockserver) + mock_endpoint = mockserver.url('/status?n=200') + TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint) + TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint) + + crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + yield crawler.crawl() self.assertEqual(crawler.spider.visited, 2) From 57824600a8295f0d8537de2a8d1c3eb9977ed36d Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Sun, 19 Aug 2018 16:56:41 +0300 Subject: [PATCH 6/6] Use six.get_unbound_function in test_same_url --- tests/test_contracts.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 0a216b745..b4209e1f6 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,5 +1,6 @@ from unittest import TextTestResult +from six import get_unbound_function from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -230,9 +231,10 @@ class ContractsManagerTest(unittest.TestCase): return TestItem() with MockServer() as mockserver: - mock_endpoint = mockserver.url('/status?n=200') - TestSameUrlSpider.parse_first.__func__.__doc__ = '@url {}'.format(mock_endpoint) - TestSameUrlSpider.parse_second.__func__.__doc__ = '@url {}'.format(mock_endpoint) + contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) + + get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc + get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) yield crawler.crawl()