From 4de493efdd80d8ff85a78009dcc120ac46b9c55c Mon Sep 17 00:00:00 2001 From: Stas Glubokiy Date: Wed, 15 Aug 2018 20:24:00 +0300 Subject: [PATCH] Add test_same_url --- tests/test_contracts.py | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b07cbee1e..e4f1a777b 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,8 +1,10 @@ from unittest import TextTestResult +from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest +from scrapy.crawler import CrawlerRunner from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request @@ -101,6 +103,29 @@ class TestSpider(Spider): pass +class TestSameUrlSpider(Spider): + + name = 'test_same_url' + + def __init__(self, *args, **kwargs): + super(TestSameUrlSpider, self).__init__(*args, **kwargs) + self.visited = 0 + + def parse_first(self, response): + """first callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + def parse_second(self, response): + """second callback + @url http://scrapy.org + """ + self.visited += 1 + return TestItem() + + class ContractsManagerTest(unittest.TestCase): contracts = [UrlContract, ReturnsContract, ScrapesContract] @@ -177,14 +202,12 @@ class ContractsManagerTest(unittest.TestCase): self.should_succeed() # scrapes_item_fail - request = self.conman.from_method(spider.scrapes_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_item_fail, self.results) request.callback(response) self.should_fail() # scrapes_dict_item_fail - request = self.conman.from_method(spider.scrapes_dict_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results) request.callback(response) self.should_fail() @@ -202,3 +225,12 @@ class ContractsManagerTest(unittest.TestCase): self.assertFalse(self.results.failures) self.assertTrue(self.results.errors) + + @defer.inlineCallbacks + def test_same_url(self): + TestSameUrlSpider.start_requests = lambda s: self.conman.from_spider(s, self.results) + + crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + yield crawler.crawl() + + self.assertEqual(crawler.spider.visited, 2)