From 4898d96337c66f7702ad2bf250c799a70557f3d7 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Sat, 26 Apr 2014 13:47:42 +0200
Subject: [PATCH] Add tests for start requests, filtered and non-filtered

---
 scrapy/tests/spiders.py    | 23 +++++++++++++++++++++++
 scrapy/tests/test_crawl.py | 17 ++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py
index 5cf15818d..3eac95d58 100644
--- a/scrapy/tests/spiders.py
+++ b/scrapy/tests/spiders.py
@@ -157,3 +157,26 @@ class SingleRequestSpider(MetaSpider):
         self.meta['failure'] = failure
         if callable(self.errback_func):
             return self.errback_func(failure)
+
+
+class DuplicateStartRequestsSpider(Spider):
+    dont_filter = True
+    name = 'duplicatestartrequests'
+    distinct_urls = 2
+    dupe_factor = 3
+
+    def start_requests(self):
+        for i in range(0, self.distinct_urls):
+            for j in range(0, self.dupe_factor):
+                url = "http://localhost:8998/echo?headers=1&body=test%d" % i
+                yield self.make_requests_from_url(url)
+
+    def make_requests_from_url(self, url):
+        return Request(url, dont_filter=self.dont_filter)
+
+    def __init__(self, url="http://localhost:8998", *args, **kwargs):
+        super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
+        self.visited = 0
+
+    def parse(self, response):
+        self.visited += 1
diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py
index a8847aa40..81666930d 100644
--- a/scrapy/tests/test_crawl.py
+++ b/scrapy/tests/test_crawl.py
@@ -5,7 +5,7 @@ from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from scrapy.utils.test import docrawl, get_testlog
 from scrapy.tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
-    BrokenStartRequestsSpider, SingleRequestSpider
+    BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
 from scrapy.tests.mockserver import MockServer
 from scrapy.http import Request
 
@@ -113,6 +113,21 @@ class CrawlTestCase(TestCase):
         #self.assertTrue(spider.seedsseen.index(None) < spider.seedsseen.index(99),
         #                spider.seedsseen)
 
+    @defer.inlineCallbacks
+    def test_start_requests_dupes(self):
+        settings = {"CONCURRENT_REQUESTS": 1}
+        spider = DuplicateStartRequestsSpider(dont_filter=True,
+                                              distinct_urls=2,
+                                              dupe_factor=3)
+        yield docrawl(spider, settings)
+        self.assertEqual(spider.visited, 6)
+
+        spider = DuplicateStartRequestsSpider(dont_filter=False,
+                                              distinct_urls=3,
+                                              dupe_factor=4)
+        yield docrawl(spider, settings)
+        self.assertEqual(spider.visited, 3)
+
     @defer.inlineCallbacks
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding