diff --git a/scrapy/tests/mockserver.py b/scrapy/tests/mockserver.py index 5c10b1793..6f58bf815 100644 --- a/scrapy/tests/mockserver.py +++ b/scrapy/tests/mockserver.py @@ -1,5 +1,5 @@ -import json, random, urllib -from time import time +import sys, time, json, random, urllib +from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.internet import reactor @@ -19,7 +19,7 @@ _request_args = { def encode_request(request): """Encode request into a JSON-serializable type""" - d = {"time": time()} + d = {"time": time.time()} for k, func in _request_args.iteritems(): d[k] = func(getattr(request, k)) return d @@ -125,6 +125,19 @@ class Root(Resource): def render(self, request): return 'Scrapy mock HTTP server\n' +class MockServer(): + + def __enter__(self): + from scrapy.utils.test import get_testenv + self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'], + stdout=PIPE, env=get_testenv()) + self.proc.stdout.readline() + + def __exit__(self, exc_type, exc_value, traceback): + self.proc.kill() + self.proc.wait() + time.sleep(0.2) + if __name__ == "__main__": root = Root() diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py new file mode 100644 index 000000000..f5c12220d --- /dev/null +++ b/scrapy/tests/spiders.py @@ -0,0 +1,55 @@ +""" +Some spiders used for testing and benchmarking +""" + +import time + +from scrapy.spider import BaseSpider +from scrapy.http import Request +from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor + +class FollowAllSpider(BaseSpider): + + name = 'follow' + link_extractor = SgmlLinkExtractor() + + def __init__(self, total=10, show=20, order="rand"): + self.urls_visited = [] + self.times = [] + url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order) + self.start_urls = [url] + + def parse(self, response): + self.urls_visited.append(response.url) + self.times.append(time.time()) + for link in self.link_extractor.extract_links(response): + yield Request(link.url, callback=self.parse) + +class DelaySpider(BaseSpider): + + name = 'delay' + + def __init__(self, n=1): + self.n = n + self.t1 = self.t2 = self.t2_err = 0 + + def start_requests(self): + self.t1 = time.time() + yield Request("http://localhost:8998/delay?n=%s" % self.n, \ + callback=self.parse, errback=self.errback) + + def parse(self, response): + self.t2 = time.time() + + def errback(self, failure): + self.t2_err = time.time() + +class SimpleSpider(BaseSpider): + + name = 'simple' + + def __init__(self, url="http://localhost:8998"): + self.start_urls = [url] + + def parse(self, response): + self.log("Got response %d" % response.status) diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index eb4e00d75..8417844e5 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -1,57 +1,9 @@ -import sys, time from twisted.internet import defer from twisted.trial.unittest import TestCase -from subprocess import Popen, PIPE -from scrapy.spider import BaseSpider -from scrapy.http import Request -from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor -from scrapy.utils.test import get_crawler, get_testenv, get_testlog +from scrapy.utils.test import get_crawler, get_testlog +from scrapy.tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider +from scrapy.tests.mockserver import MockServer -class FollowAllSpider(BaseSpider): - - name = 'follow' - link_extractor = SgmlLinkExtractor() - - def __init__(self, total=10, show=20, order="rand"): - self.urls_visited = [] - self.times = [] - url = "http://localhost:8998/follow?total=%s&show=%s&order=%s" % (total, show, order) - self.start_urls = [url] - - def parse(self, response): - self.urls_visited.append(response.url) - self.times.append(time.time()) - for link in self.link_extractor.extract_links(response): - yield Request(link.url, callback=self.parse) - -class DelaySpider(BaseSpider): - - name = 'delay' - - def __init__(self, n=1): - self.n = n - self.t1 = self.t2 = self.t2_err = 0 - - def start_requests(self): - self.t1 = time.time() - yield Request("http://localhost:8998/delay?n=%s" % self.n, \ - callback=self.parse, errback=self.errback) - - def parse(self, response): - self.t2 = time.time() - - def errback(self, failure): - self.t2_err = time.time() - -class SimpleSpider(BaseSpider): - - name = 'simple' - - def __init__(self, url="http://localhost:8998"): - self.start_urls = [url] - - def parse(self, response): - self.log("Got response %d" % response.status) def docrawl(spider, settings=None): crawler = get_crawler(settings) @@ -62,14 +14,11 @@ def docrawl(spider, settings=None): class CrawlTestCase(TestCase): def setUp(self): - self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'], - stdout=PIPE, env=get_testenv()) - self.proc.stdout.readline() + self.mockserver = MockServer() + self.mockserver.__enter__() def tearDown(self): - self.proc.kill() - self.proc.wait() - time.sleep(0.2) + self.mockserver.__exit__(None, None, None) @defer.inlineCallbacks def test_follow_all(self):