added context manager for mock server, moved test spiders into a separate module (scrapy.tests.spiders)

scrapy/tests/mockserver.py

@@ -1,5 +1,5 @@
-import json, random, urllib
-from time import time
+import sys, time, json, random, urllib
+from subprocess import Popen, PIPE
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.internet import reactor
@@ -19,7 +19,7 @@ _request_args = {
 
 def encode_request(request):
     """Encode request into a JSON-serializable type"""
-    d = {"time": time()}
+    d = {"time": time.time()}
     for k, func in _request_args.iteritems():
         d[k] = func(getattr(request, k))
     return d
@@ -125,6 +125,19 @@ class Root(Resource):
     def render(self, request):
         return 'Scrapy mock HTTP server\n'
 
+class MockServer():
+
+    def __enter__(self):
+        from scrapy.utils.test import get_testenv
+        self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'],
+            stdout=PIPE, env=get_testenv())
+        self.proc.stdout.readline()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.proc.kill()
+        self.proc.wait()
+        time.sleep(0.2)
+
 
 if __name__ == "__main__":
     root = Root()

scrapy/tests/spiders.py

@@ -0,0 +1,55 @@
+"""
+Some spiders used for testing and benchmarking
+"""
+
+import time
+
+from scrapy.spider import BaseSpider
+from scrapy.http import Request
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+
+class FollowAllSpider(BaseSpider):
+
+    name = 'follow'
+    link_extractor = SgmlLinkExtractor()
+
+    def __init__(self, total=10, show=20, order="rand"):
+        self.urls_visited = []
+        self.times = []
+        url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order)
+        self.start_urls = [url]
+
+    def parse(self, response):
+        self.urls_visited.append(response.url)
+        self.times.append(time.time())
+        for link in self.link_extractor.extract_links(response):
+            yield Request(link.url, callback=self.parse)
+
+class DelaySpider(BaseSpider):
+
+    name = 'delay'
+
+    def __init__(self, n=1):
+        self.n = n
+        self.t1 = self.t2 = self.t2_err = 0
+
+    def start_requests(self):
+        self.t1 = time.time()
+        yield Request("http://localhost:8998/delay?n=%s" % self.n, \
+            callback=self.parse, errback=self.errback)
+
+    def parse(self, response):
+        self.t2 = time.time()
+
+    def errback(self, failure):
+        self.t2_err = time.time()
+
+class SimpleSpider(BaseSpider):
+
+    name = 'simple'
+
+    def __init__(self, url="http://localhost:8998"):
+        self.start_urls = [url]
+
+    def parse(self, response):
+        self.log("Got response %d" % response.status)

scrapy/tests/test_crawl.py

@@ -1,57 +1,9 @@
-import sys, time
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
-from subprocess import Popen, PIPE
-from scrapy.spider import BaseSpider
-from scrapy.http import Request
-from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
-from scrapy.utils.test import get_crawler, get_testenv, get_testlog
+from scrapy.utils.test import get_crawler, get_testlog
+from scrapy.tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider
+from scrapy.tests.mockserver import MockServer
 
-class FollowAllSpider(BaseSpider):
-
-    name = 'follow'
-    link_extractor = SgmlLinkExtractor()
-
-    def __init__(self, total=10, show=20, order="rand"):
-        self.urls_visited = []
-        self.times = []
-        url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order)
-        self.start_urls = [url]
-
-    def parse(self, response):
-        self.urls_visited.append(response.url)
-        self.times.append(time.time())
-        for link in self.link_extractor.extract_links(response):
-            yield Request(link.url, callback=self.parse)
-
-class DelaySpider(BaseSpider):
-
-    name = 'delay'
-
-    def __init__(self, n=1):
-        self.n = n
-        self.t1 = self.t2 = self.t2_err = 0
-
-    def start_requests(self):
-        self.t1 = time.time()
-        yield Request("http://localhost:8998/delay?n=%s" % self.n, \
-            callback=self.parse, errback=self.errback)
-
-    def parse(self, response):
-        self.t2 = time.time()
-
-    def errback(self, failure):
-        self.t2_err = time.time()
-
-class SimpleSpider(BaseSpider):
-
-    name = 'simple'
-
-    def __init__(self, url="http://localhost:8998"):
-        self.start_urls = [url]
-
-    def parse(self, response):
-        self.log("Got response %d" % response.status)
 
 def docrawl(spider, settings=None):
     crawler = get_crawler(settings)
@@ -62,14 +14,11 @@ def docrawl(spider, settings=None):
 class CrawlTestCase(TestCase):
 
     def setUp(self):
-        self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'],
-            stdout=PIPE, env=get_testenv())
-        self.proc.stdout.readline()
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
 
     def tearDown(self):
-        self.proc.kill()
-        self.proc.wait()
-        time.sleep(0.2)
+        self.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
     def test_follow_all(self):