diff --git a/tests/mockserver.py b/tests/mockserver.py index 98723846e..f36ce3c44 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -192,9 +192,15 @@ class MockServer(): def __enter__(self): from scrapy.utils.test import get_testenv + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], stdout=PIPE, env=get_testenv()) - self.proc.stdout.readline() + http_address = self.proc.stdout.readline().strip().decode('ascii') + https_address = self.proc.stdout.readline().strip().decode('ascii') + + self.http_address = http_address + self.https_address = https_address + return self def __exit__(self, exc_type, exc_value, traceback): @@ -202,6 +208,12 @@ class MockServer(): self.proc.wait() time.sleep(0.2) + def url(self, path, is_secure=False): + host = self.http_address + if is_secure: + host = self.https_address + return host + path + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'): return ssl.DefaultOpenSSLContextFactory( @@ -213,14 +225,17 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": root = Root() factory = Site(root) - httpPort = reactor.listenTCP(8998, factory) + httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(8999, factory, contextFactory) + httpsPort = reactor.listenSSL(0, factory, contextFactory) def print_listening(): httpHost = httpPort.getHost() httpsHost = httpsPort.getHost() - print("Mock server running at http://%s:%d and https://%s:%d" % ( - httpHost.host, httpHost.port, httpsHost.host, httpsHost.port)) + httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) + httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/spiders.py b/tests/spiders.py index 1038b69de..7816bf7c7 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -11,7 +11,12 @@ from scrapy.item import Item from scrapy.linkextractors import LinkExtractor -class MetaSpider(Spider): +class MockServerSpider(Spider): + def __init__(self, mockserver=None, *args, **kwargs): + super(MockServerSpider, self).__init__(*args, **kwargs) + self.mockserver = mockserver + +class MetaSpider(MockServerSpider): name = 'meta' @@ -33,7 +38,7 @@ class FollowAllSpider(MetaSpider): self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1)) self.start_urls = [url] def parse(self, response): @@ -55,7 +60,7 @@ class DelaySpider(MetaSpider): def start_requests(self): self.t1 = time.time() - url = "http://localhost:8998/delay?n=%s&b=%s" % (self.n, self.b) + url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b)) yield Request(url, callback=self.parse, errback=self.errback) def parse(self, response): @@ -121,7 +126,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): for s in range(100): qargs = {'total': 10, 'seed': s} - url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1) + url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1) yield Request(url, meta={'seed': s}) if self.fail_yielding: 2 / 0 @@ -160,7 +165,7 @@ class SingleRequestSpider(MetaSpider): return self.errback_func(failure) -class DuplicateStartRequestsSpider(Spider): +class DuplicateStartRequestsSpider(MockServerSpider): dont_filter = True name = 'duplicatestartrequests' distinct_urls = 2 @@ -169,7 +174,7 @@ class DuplicateStartRequestsSpider(Spider): def start_requests(self): for i in range(0, self.distinct_urls): for j in range(0, self.dupe_factor): - url = "http://localhost:8998/echo?headers=1&body=test%d" % i + url = self.mockserver.url("/echo?headers=1&body=test%d" % i) yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index fa0b48998..0eb1b7944 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -18,7 +18,7 @@ class TestCloseSpider(TestCase): def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_itemcount') itemcount = crawler.stats.get_value('item_scraped_count') @@ -28,7 +28,7 @@ class TestCloseSpider(TestCase): def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on}) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_pagecount') pagecount = crawler.stats.get_value('response_received_count') @@ -38,7 +38,7 @@ class TestCloseSpider(TestCase): def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_errorcount') key = 'spider_exceptions/{name}'\ @@ -50,7 +50,7 @@ class TestCloseSpider(TestCase): def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on}) - yield crawler.crawl(total=1000000) + yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') stats = crawler.stats diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d5babdded..3fc13eeb7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -26,7 +26,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_follow_all(self): crawler = self.runner.create_crawler(FollowAllSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks @@ -42,7 +42,7 @@ class CrawlTestCase(TestCase): def _test_delay(self, delay, randomize): settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize} crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider) - yield crawler.crawl(maxlatency=delay * 2) + yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver) t = crawler.spider.times totaltime = t[-1] - t[0] avgd = totaltime / (len(t) - 1) @@ -53,7 +53,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_success(self): crawler = self.runner.create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 > 0) self.assertTrue(crawler.spider.t2 > crawler.spider.t1) @@ -61,13 +61,13 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_timeout_failure(self): crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider) - yield crawler.crawl(n=0.5) + yield crawler.crawl(n=0.5, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) self.assertTrue(crawler.spider.t2_err > crawler.spider.t1) # server hangs after receiving response headers - yield crawler.crawl(n=0.5, b=1) + yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) self.assertTrue(crawler.spider.t1 > 0) self.assertTrue(crawler.spider.t2 == 0) self.assertTrue(crawler.spider.t2_err > 0) @@ -77,14 +77,14 @@ class CrawlTestCase(TestCase): def test_retry_503(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/status?n=503") + yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_retry_conn_failed(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:65432/status?n=503") + yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -92,14 +92,14 @@ class CrawlTestCase(TestCase): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: # try to fetch the homepage of a non-existent domain - yield crawler.crawl("http://dns.resolution.invalid./") + yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks def test_start_requests_bug_before_yield(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_before_yield=1) + yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -110,7 +110,7 @@ class CrawlTestCase(TestCase): def test_start_requests_bug_yielding(self): with LogCapture('scrapy', level=logging.ERROR) as l: crawler = self.runner.create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl(fail_yielding=1) + yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) self.assertEqual(len(l.records), 1) record = l.records[0] @@ -121,7 +121,7 @@ class CrawlTestCase(TestCase): def test_start_requests_lazyness(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) #self.assertTrue(False, crawler.spider.seedsseen) #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99), # crawler.spider.seedsseen) @@ -130,10 +130,10 @@ class CrawlTestCase(TestCase): def test_start_requests_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider) - yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3) + yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 6) - yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4) + yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver) self.assertEqual(crawler.spider.visited, 3) @defer.inlineCallbacks @@ -160,7 +160,7 @@ with multiples lines '''}) crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/raw?{0}".format(query)) + yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver) self.assertEqual(str(l).count("Got response 200"), 1) @defer.inlineCallbacks @@ -168,7 +168,7 @@ with multiples lines # connection lost after receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=0") + yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver) self._assert_retried(l) @defer.inlineCallbacks @@ -176,7 +176,7 @@ with multiples lines # connection lost before receiving data crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("http://localhost:8998/drop?abort=1") + yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver) self._assert_retried(l) def _assert_retried(self, log): @@ -186,7 +186,7 @@ with multiples lines @defer.inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1) + req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1) req1 = req0.replace() req2 = req0.replace(headers={'Referer': None}) req3 = req0.replace(headers={'Referer': 'http://example.com'}) @@ -194,7 +194,7 @@ with multiples lines req1.meta['next'] = req2 req2.meta['next'] = req3 crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0) + yield crawler.crawl(seed=req0, mockserver=self.mockserver) # basic asserts in case of weird communication errors self.assertIn('responses', crawler.spider.meta) self.assertNotIn('failures', crawler.spider.meta) @@ -220,7 +220,7 @@ with multiples lines est.append(get_engine_status(crawler.engine)) crawler = self.runner.create_crawler(SingleRequestSpider) - yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb) + yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver) self.assertEqual(len(est), 1, est) s = dict(est[0]) self.assertEqual(s['engine.spider.name'], crawler.spider.name) @@ -244,7 +244,7 @@ with multiples lines raise TestError crawler = self.runner.create_crawler(FaultySpider) - yield self.assertFailure(crawler.crawl(), TestError) + yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError) self.assertFalse(crawler.crawling) @defer.inlineCallbacks @@ -256,7 +256,7 @@ with multiples lines } crawler = CrawlerRunner(settings).create_crawler(SimpleSpider) yield self.assertFailure( - self.runner.crawl(crawler, "http://localhost:8998/status?n=200"), + self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver), ZeroDivisionError) self.assertFalse(crawler.crawling) @@ -264,13 +264,13 @@ with multiples lines def test_crawlerrunner_accepts_crawler(self): crawler = self.runner.create_crawler(SimpleSpider) with LogCapture() as log: - yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200") + yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) self.assertIn("Got response 200", str(log)) @defer.inlineCallbacks def test_crawl_multiple(self): - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200") - self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503") + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver) + self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver) with LogCapture() as log: yield self.runner.join() diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index b34faa7e7..c91be2c0c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -534,14 +534,14 @@ class Http11MockServerTestCase(unittest.TestCase): crawler = get_crawler(SingleRequestSpider) # http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid # download it - yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000})) + yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000})) failure = crawler.spider.meta['failure'] self.assertIsInstance(failure.value, defer.CancelledError) @defer.inlineCallbacks def test_download(self): crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=Request(url='http://localhost:8998')) + yield crawler.crawl(seed=Request(url=self.mockserver.url(''))) failure = crawler.spider.meta.get('failure') self.assertTrue(failure == None) reason = crawler.spider.meta['close_reason'] @@ -551,7 +551,7 @@ class Http11MockServerTestCase(unittest.TestCase): def test_download_gzip_response(self): crawler = get_crawler(SingleRequestSpider) body = b'1' * 100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', + request = Request(self.mockserver.url('/payload'), method='POST', body=body, meta={'download_maxsize': 50}) yield crawler.crawl(seed=request) failure = crawler.spider.meta['failure'] @@ -560,7 +560,7 @@ class Http11MockServerTestCase(unittest.TestCase): if six.PY2: request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') + request = request.replace(url=self.mockserver.url('/xpayload')) yield crawler.crawl(seed=request) # download_maxsize = 50 is enough for the gzipped response failure = crawler.spider.meta.get('failure') diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f55927121..0d9f1e83c 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -179,6 +179,7 @@ class FeedExportTest(unittest.TestCase): try: with MockServer() as s: runner = CrawlerRunner(Settings(defaults)) + spider_cls.start_urls = [s.url('/')] yield runner.crawl(spider_cls) with open(res_name, 'rb') as f: @@ -194,7 +195,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): for item in items: @@ -210,7 +210,6 @@ class FeedExportTest(unittest.TestCase): """ class TestSpider(scrapy.Spider): name = 'testspider' - start_urls = ['http://localhost:8998/'] def parse(self, response): pass diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 9b81f827d..5985a6f3e 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -46,7 +46,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): def _process_url(self, url): return add_or_replace_parameter( - 'http://localhost:8998/redirect-to', + self.mockserver.url('/redirect-to'), 'goto', url) @@ -134,7 +134,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -143,7 +143,7 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -152,9 +152,10 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_download_failure(crawler, self.items, 302, str(log)) @defer.inlineCallbacks @@ -165,9 +166,10 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl("http://localhost:8998/files/images/", + yield crawler.crawl(self.mockserver.url("/files/images/"), media_key=self.media_key, - media_urls_key=self.media_urls_key) + media_urls_key=self.media_urls_key, + mockserver=self.mockserver) self._assert_files_downloaded(self.items, str(log)) self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 6213a51e8..ae1236bcb 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,6 +2,7 @@ import json import os import time +from six.moves.urllib.parse import urlsplit, urlunsplit from threading import Thread from libmproxy import controller, proxy from netlib import http_auth @@ -17,7 +18,7 @@ from tests.mockserver import MockServer class HTTPSProxy(controller.Master, Thread): - def __init__(self, port): + def __init__(self): password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), @@ -25,10 +26,19 @@ class HTTPSProxy(controller.Master, Thread): server = proxy.ProxyServer(proxy.ProxyConfig( authenticator = authenticator, cacert = cert_path), - port) + 0) + self.server = server Thread.__init__(self) controller.Master.__init__(self, server) + def http_address(self): + return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() + + +def _wrong_credentials(proxy_url): + bad_auth_proxy = list(urlsplit(proxy_url)) + bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') + return urlunsplit(bad_auth_proxy) class ProxyConnectTestCase(TestCase): @@ -36,12 +46,14 @@ class ProxyConnectTestCase(TestCase): self.mockserver = MockServer() self.mockserver.__enter__() self._oldenv = os.environ.copy() - self._proxy = HTTPSProxy(8888) + + self._proxy = HTTPSProxy() self._proxy.start() + # Wait for the proxy to start. time.sleep(1.0) - os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + os.environ['https_proxy'] = self._proxy.http_address() + os.environ['http_proxy'] = self._proxy.http_address() def tearDown(self): self.mockserver.__exit__(None, None, None) @@ -52,17 +64,17 @@ class ProxyConnectTestCase(TestCase): def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) @defer.inlineCallbacks def test_https_noconnect(self): - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_connect_tunnel_error(self): @@ -73,18 +85,17 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_tunnel_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. self._assert_got_tunnel_error(l) - os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): - request = Request("https://localhost:8999/echo") + request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) with LogCapture() as l: yield crawler.crawl(seed=request) @@ -94,10 +105,10 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' + os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) with LogCapture() as l: - yield crawler.crawl("https://localhost:8999/status?n=200") + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 19e6bbdcd..dacd0147f 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -11,20 +11,21 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError from scrapy.settings import Settings +from tests.spiders import MockServerSpider -class _HttpErrorSpider(Spider): +class _HttpErrorSpider(MockServerSpider): name = 'httperror' - start_urls = [ - "http://localhost:8998/status?n=200", - "http://localhost:8998/status?n=404", - "http://localhost:8998/status?n=402", - "http://localhost:8998/status?n=500", - ] bypass_status_codes = set() def __init__(self, *args, **kwargs): super(_HttpErrorSpider, self).__init__(*args, **kwargs) + self.start_urls = [ + self.mockserver.url("/status?n=200"), + self.mockserver.url("/status?n=404"), + self.mockserver.url("/status?n=402"), + self.mockserver.url("/status?n=500"), + ] self.failed = set() self.skipped = set() self.parsed = set() @@ -169,7 +170,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): @defer.inlineCallbacks def test_middleware_works(self): crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) assert not crawler.spider.skipped, crawler.spider.skipped self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -184,7 +185,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def test_logging(self): crawler = get_crawler(_HttpErrorSpider) with LogCapture() as log: - yield crawler.crawl(bypass_status_codes={402}) + yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) self.assertEqual(crawler.spider.parsed, {'200', '402'}) self.assertEqual(crawler.spider.skipped, {'402'}) self.assertEqual(crawler.spider.failed, {'404', '500'}) @@ -199,7 +200,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.INFO) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) @@ -211,7 +212,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): # with level WARNING, we shouldn't capture anything from HttpError crawler = get_crawler(_HttpErrorSpider) with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'})