mirror of https://github.com/scrapy/scrapy.git
[MRG+1] Automatic port selection for servicies in unit tests (#3210)
* ability to pass port as a parameter * try to find free ports * use environment variables to pass mock server address * get mock server address from environment variables * ability to select ports for proxy in runtime * use common method for URLs from mock server * https support * get mock server address * get mock address * replace hand-written mechanism by kernel-based one * use ephemeral ports in mockserver * strip EOL from addresses * use ephemeral port in proxy * no need to restore environment as it is restored in tearDown * decode bytes * use mockserver address as a variable * ability to pass address as variable * per test-case mockserver * use base class * remove obsolete environment manipulation * return usage of proxy for http cases * common method for broking proxy auth credentials * python version-independent url methods
This commit is contained in:
Vostretsov Nikita
Konstantin Lopuhin
parent
bac1e2d47d
commit
b364d27247
|
|
@ -192,9 +192,15 @@ class MockServer():
|
|||
|
||||
def __enter__(self):
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
|
||||
stdout=PIPE, env=get_testenv())
|
||||
self.proc.stdout.readline()
|
||||
http_address = self.proc.stdout.readline().strip().decode('ascii')
|
||||
https_address = self.proc.stdout.readline().strip().decode('ascii')
|
||||
|
||||
self.http_address = http_address
|
||||
self.https_address = https_address
|
||||
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
|
|
@ -202,6 +208,12 @@ class MockServer():
|
|||
self.proc.wait()
|
||||
time.sleep(0.2)
|
||||
|
||||
def url(self, path, is_secure=False):
|
||||
host = self.http_address
|
||||
if is_secure:
|
||||
host = self.https_address
|
||||
return host + path
|
||||
|
||||
|
||||
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt'):
|
||||
return ssl.DefaultOpenSSLContextFactory(
|
||||
|
|
@ -213,14 +225,17 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
|
|||
if __name__ == "__main__":
|
||||
root = Root()
|
||||
factory = Site(root)
|
||||
httpPort = reactor.listenTCP(8998, factory)
|
||||
httpPort = reactor.listenTCP(0, factory)
|
||||
contextFactory = ssl_context_factory()
|
||||
httpsPort = reactor.listenSSL(8999, factory, contextFactory)
|
||||
httpsPort = reactor.listenSSL(0, factory, contextFactory)
|
||||
|
||||
def print_listening():
|
||||
httpHost = httpPort.getHost()
|
||||
httpsHost = httpsPort.getHost()
|
||||
print("Mock server running at http://%s:%d and https://%s:%d" % (
|
||||
httpHost.host, httpHost.port, httpsHost.host, httpsHost.port))
|
||||
httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
|
||||
httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
|
||||
print(httpAddress)
|
||||
print(httpsAddress)
|
||||
|
||||
reactor.callWhenRunning(print_listening)
|
||||
reactor.run()
|
||||
|
|
|
|||
|
|
@ -11,7 +11,12 @@ from scrapy.item import Item
|
|||
from scrapy.linkextractors import LinkExtractor
|
||||
|
||||
|
||||
class MetaSpider(Spider):
|
||||
class MockServerSpider(Spider):
|
||||
def __init__(self, mockserver=None, *args, **kwargs):
|
||||
super(MockServerSpider, self).__init__(*args, **kwargs)
|
||||
self.mockserver = mockserver
|
||||
|
||||
class MetaSpider(MockServerSpider):
|
||||
|
||||
name = 'meta'
|
||||
|
||||
|
|
@ -33,7 +38,7 @@ class FollowAllSpider(MetaSpider):
|
|||
self.urls_visited = []
|
||||
self.times = []
|
||||
qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
|
||||
url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
|
||||
url = self.mockserver.url("/follow?%s" % urlencode(qargs, doseq=1))
|
||||
self.start_urls = [url]
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -55,7 +60,7 @@ class DelaySpider(MetaSpider):
|
|||
|
||||
def start_requests(self):
|
||||
self.t1 = time.time()
|
||||
url = "http://localhost:8998/delay?n=%s&b=%s" % (self.n, self.b)
|
||||
url = self.mockserver.url("/delay?n=%s&b=%s" % (self.n, self.b))
|
||||
yield Request(url, callback=self.parse, errback=self.errback)
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -121,7 +126,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
|
|||
|
||||
for s in range(100):
|
||||
qargs = {'total': 10, 'seed': s}
|
||||
url = "http://localhost:8998/follow?%s" % urlencode(qargs, doseq=1)
|
||||
url = self.mockserver.url("/follow?%s") % urlencode(qargs, doseq=1)
|
||||
yield Request(url, meta={'seed': s})
|
||||
if self.fail_yielding:
|
||||
2 / 0
|
||||
|
|
@ -160,7 +165,7 @@ class SingleRequestSpider(MetaSpider):
|
|||
return self.errback_func(failure)
|
||||
|
||||
|
||||
class DuplicateStartRequestsSpider(Spider):
|
||||
class DuplicateStartRequestsSpider(MockServerSpider):
|
||||
dont_filter = True
|
||||
name = 'duplicatestartrequests'
|
||||
distinct_urls = 2
|
||||
|
|
@ -169,7 +174,7 @@ class DuplicateStartRequestsSpider(Spider):
|
|||
def start_requests(self):
|
||||
for i in range(0, self.distinct_urls):
|
||||
for j in range(0, self.dupe_factor):
|
||||
url = "http://localhost:8998/echo?headers=1&body=test%d" % i
|
||||
url = self.mockserver.url("/echo?headers=1&body=test%d" % i)
|
||||
yield Request(url, dont_filter=self.dont_filter)
|
||||
|
||||
def __init__(self, url="http://localhost:8998", *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class TestCloseSpider(TestCase):
|
|||
def test_closespider_itemcount(self):
|
||||
close_on = 5
|
||||
crawler = get_crawler(ItemSpider, {'CLOSESPIDER_ITEMCOUNT': close_on})
|
||||
yield crawler.crawl()
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_itemcount')
|
||||
itemcount = crawler.stats.get_value('item_scraped_count')
|
||||
|
|
@ -28,7 +28,7 @@ class TestCloseSpider(TestCase):
|
|||
def test_closespider_pagecount(self):
|
||||
close_on = 5
|
||||
crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_PAGECOUNT': close_on})
|
||||
yield crawler.crawl()
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_pagecount')
|
||||
pagecount = crawler.stats.get_value('response_received_count')
|
||||
|
|
@ -38,7 +38,7 @@ class TestCloseSpider(TestCase):
|
|||
def test_closespider_errorcount(self):
|
||||
close_on = 5
|
||||
crawler = get_crawler(ErrorSpider, {'CLOSESPIDER_ERRORCOUNT': close_on})
|
||||
yield crawler.crawl(total=1000000)
|
||||
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_errorcount')
|
||||
key = 'spider_exceptions/{name}'\
|
||||
|
|
@ -50,7 +50,7 @@ class TestCloseSpider(TestCase):
|
|||
def test_closespider_timeout(self):
|
||||
close_on = 0.1
|
||||
crawler = get_crawler(FollowAllSpider, {'CLOSESPIDER_TIMEOUT': close_on})
|
||||
yield crawler.crawl(total=1000000)
|
||||
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertEqual(reason, 'closespider_timeout')
|
||||
stats = crawler.stats
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ class CrawlTestCase(TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_follow_all(self):
|
||||
crawler = self.runner.create_crawler(FollowAllSpider)
|
||||
yield crawler.crawl()
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -42,7 +42,7 @@ class CrawlTestCase(TestCase):
|
|||
def _test_delay(self, delay, randomize):
|
||||
settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
|
||||
crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
|
||||
yield crawler.crawl(maxlatency=delay * 2)
|
||||
yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver)
|
||||
t = crawler.spider.times
|
||||
totaltime = t[-1] - t[0]
|
||||
avgd = totaltime / (len(t) - 1)
|
||||
|
|
@ -53,7 +53,7 @@ class CrawlTestCase(TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_timeout_success(self):
|
||||
crawler = self.runner.create_crawler(DelaySpider)
|
||||
yield crawler.crawl(n=0.5)
|
||||
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
|
||||
self.assertTrue(crawler.spider.t1 > 0)
|
||||
self.assertTrue(crawler.spider.t2 > 0)
|
||||
self.assertTrue(crawler.spider.t2 > crawler.spider.t1)
|
||||
|
|
@ -61,13 +61,13 @@ class CrawlTestCase(TestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_timeout_failure(self):
|
||||
crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
|
||||
yield crawler.crawl(n=0.5)
|
||||
yield crawler.crawl(n=0.5, mockserver=self.mockserver)
|
||||
self.assertTrue(crawler.spider.t1 > 0)
|
||||
self.assertTrue(crawler.spider.t2 == 0)
|
||||
self.assertTrue(crawler.spider.t2_err > 0)
|
||||
self.assertTrue(crawler.spider.t2_err > crawler.spider.t1)
|
||||
# server hangs after receiving response headers
|
||||
yield crawler.crawl(n=0.5, b=1)
|
||||
yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)
|
||||
self.assertTrue(crawler.spider.t1 > 0)
|
||||
self.assertTrue(crawler.spider.t2 == 0)
|
||||
self.assertTrue(crawler.spider.t2_err > 0)
|
||||
|
|
@ -77,14 +77,14 @@ class CrawlTestCase(TestCase):
|
|||
def test_retry_503(self):
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("http://localhost:8998/status?n=503")
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
|
||||
self._assert_retried(l)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_retry_conn_failed(self):
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("http://localhost:65432/status?n=503")
|
||||
yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
|
||||
self._assert_retried(l)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -92,14 +92,14 @@ class CrawlTestCase(TestCase):
|
|||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
# try to fetch the homepage of a non-existent domain
|
||||
yield crawler.crawl("http://dns.resolution.invalid./")
|
||||
yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
|
||||
self._assert_retried(l)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_start_requests_bug_before_yield(self):
|
||||
with LogCapture('scrapy', level=logging.ERROR) as l:
|
||||
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
|
||||
yield crawler.crawl(fail_before_yield=1)
|
||||
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
|
||||
|
||||
self.assertEqual(len(l.records), 1)
|
||||
record = l.records[0]
|
||||
|
|
@ -110,7 +110,7 @@ class CrawlTestCase(TestCase):
|
|||
def test_start_requests_bug_yielding(self):
|
||||
with LogCapture('scrapy', level=logging.ERROR) as l:
|
||||
crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
|
||||
yield crawler.crawl(fail_yielding=1)
|
||||
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
|
||||
|
||||
self.assertEqual(len(l.records), 1)
|
||||
record = l.records[0]
|
||||
|
|
@ -121,7 +121,7 @@ class CrawlTestCase(TestCase):
|
|||
def test_start_requests_lazyness(self):
|
||||
settings = {"CONCURRENT_REQUESTS": 1}
|
||||
crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
|
||||
yield crawler.crawl()
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
#self.assertTrue(False, crawler.spider.seedsseen)
|
||||
#self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
|
||||
# crawler.spider.seedsseen)
|
||||
|
|
@ -130,10 +130,10 @@ class CrawlTestCase(TestCase):
|
|||
def test_start_requests_dupes(self):
|
||||
settings = {"CONCURRENT_REQUESTS": 1}
|
||||
crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
|
||||
yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3)
|
||||
yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.visited, 6)
|
||||
|
||||
yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4)
|
||||
yield crawler.crawl(dont_filter=False, distinct_urls=3, dupe_factor=4, mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.visited, 3)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -160,7 +160,7 @@ with multiples lines
|
|||
'''})
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("http://localhost:8998/raw?{0}".format(query))
|
||||
yield crawler.crawl(self.mockserver.url("/raw?{0}".format(query)), mockserver=self.mockserver)
|
||||
self.assertEqual(str(l).count("Got response 200"), 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -168,7 +168,7 @@ with multiples lines
|
|||
# connection lost after receiving data
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("http://localhost:8998/drop?abort=0")
|
||||
yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
|
||||
self._assert_retried(l)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -176,7 +176,7 @@ with multiples lines
|
|||
# connection lost before receiving data
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("http://localhost:8998/drop?abort=1")
|
||||
yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
|
||||
self._assert_retried(l)
|
||||
|
||||
def _assert_retried(self, log):
|
||||
|
|
@ -186,7 +186,7 @@ with multiples lines
|
|||
@defer.inlineCallbacks
|
||||
def test_referer_header(self):
|
||||
"""Referer header is set by RefererMiddleware unless it is already set"""
|
||||
req0 = Request('http://localhost:8998/echo?headers=1&body=0', dont_filter=1)
|
||||
req0 = Request(self.mockserver.url('/echo?headers=1&body=0'), dont_filter=1)
|
||||
req1 = req0.replace()
|
||||
req2 = req0.replace(headers={'Referer': None})
|
||||
req3 = req0.replace(headers={'Referer': 'http://example.com'})
|
||||
|
|
@ -194,7 +194,7 @@ with multiples lines
|
|||
req1.meta['next'] = req2
|
||||
req2.meta['next'] = req3
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=req0)
|
||||
yield crawler.crawl(seed=req0, mockserver=self.mockserver)
|
||||
# basic asserts in case of weird communication errors
|
||||
self.assertIn('responses', crawler.spider.meta)
|
||||
self.assertNotIn('failures', crawler.spider.meta)
|
||||
|
|
@ -220,7 +220,7 @@ with multiples lines
|
|||
est.append(get_engine_status(crawler.engine))
|
||||
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed='http://localhost:8998/', callback_func=cb)
|
||||
yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
|
||||
self.assertEqual(len(est), 1, est)
|
||||
s = dict(est[0])
|
||||
self.assertEqual(s['engine.spider.name'], crawler.spider.name)
|
||||
|
|
@ -244,7 +244,7 @@ with multiples lines
|
|||
raise TestError
|
||||
|
||||
crawler = self.runner.create_crawler(FaultySpider)
|
||||
yield self.assertFailure(crawler.crawl(), TestError)
|
||||
yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
|
||||
self.assertFalse(crawler.crawling)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -256,7 +256,7 @@ with multiples lines
|
|||
}
|
||||
crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
|
||||
yield self.assertFailure(
|
||||
self.runner.crawl(crawler, "http://localhost:8998/status?n=200"),
|
||||
self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
|
||||
ZeroDivisionError)
|
||||
self.assertFalse(crawler.crawling)
|
||||
|
||||
|
|
@ -264,13 +264,13 @@ with multiples lines
|
|||
def test_crawlerrunner_accepts_crawler(self):
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as log:
|
||||
yield self.runner.crawl(crawler, "http://localhost:8998/status?n=200")
|
||||
yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
|
||||
self.assertIn("Got response 200", str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_crawl_multiple(self):
|
||||
self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=200")
|
||||
self.runner.crawl(SimpleSpider, "http://localhost:8998/status?n=503")
|
||||
self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
|
||||
self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
|
||||
|
||||
with LogCapture() as log:
|
||||
yield self.runner.join()
|
||||
|
|
|
|||
|
|
@ -534,14 +534,14 @@ class Http11MockServerTestCase(unittest.TestCase):
|
|||
crawler = get_crawler(SingleRequestSpider)
|
||||
# http://localhost:8998/partial set Content-Length to 1024, use download_maxsize= 1000 to avoid
|
||||
# download it
|
||||
yield crawler.crawl(seed=Request(url='http://localhost:8998/partial', meta={'download_maxsize': 1000}))
|
||||
yield crawler.crawl(seed=Request(url=self.mockserver.url('/partial'), meta={'download_maxsize': 1000}))
|
||||
failure = crawler.spider.meta['failure']
|
||||
self.assertIsInstance(failure.value, defer.CancelledError)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_download(self):
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=Request(url='http://localhost:8998'))
|
||||
yield crawler.crawl(seed=Request(url=self.mockserver.url('')))
|
||||
failure = crawler.spider.meta.get('failure')
|
||||
self.assertTrue(failure == None)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
|
|
@ -551,7 +551,7 @@ class Http11MockServerTestCase(unittest.TestCase):
|
|||
def test_download_gzip_response(self):
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
body = b'1' * 100 # PayloadResource requires body length to be 100
|
||||
request = Request('http://localhost:8998/payload', method='POST',
|
||||
request = Request(self.mockserver.url('/payload'), method='POST',
|
||||
body=body, meta={'download_maxsize': 50})
|
||||
yield crawler.crawl(seed=request)
|
||||
failure = crawler.spider.meta['failure']
|
||||
|
|
@ -560,7 +560,7 @@ class Http11MockServerTestCase(unittest.TestCase):
|
|||
|
||||
if six.PY2:
|
||||
request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
|
||||
request = request.replace(url='http://localhost:8998/xpayload')
|
||||
request = request.replace(url=self.mockserver.url('/xpayload'))
|
||||
yield crawler.crawl(seed=request)
|
||||
# download_maxsize = 50 is enough for the gzipped response
|
||||
failure = crawler.spider.meta.get('failure')
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ class FeedExportTest(unittest.TestCase):
|
|||
try:
|
||||
with MockServer() as s:
|
||||
runner = CrawlerRunner(Settings(defaults))
|
||||
spider_cls.start_urls = [s.url('/')]
|
||||
yield runner.crawl(spider_cls)
|
||||
|
||||
with open(res_name, 'rb') as f:
|
||||
|
|
@ -194,7 +195,6 @@ class FeedExportTest(unittest.TestCase):
|
|||
"""
|
||||
class TestSpider(scrapy.Spider):
|
||||
name = 'testspider'
|
||||
start_urls = ['http://localhost:8998/']
|
||||
|
||||
def parse(self, response):
|
||||
for item in items:
|
||||
|
|
@ -210,7 +210,6 @@ class FeedExportTest(unittest.TestCase):
|
|||
"""
|
||||
class TestSpider(scrapy.Spider):
|
||||
name = 'testspider'
|
||||
start_urls = ['http://localhost:8998/']
|
||||
|
||||
def parse(self, response):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
|
|||
|
||||
def _process_url(self, url):
|
||||
return add_or_replace_parameter(
|
||||
'http://localhost:8998/redirect-to',
|
||||
self.mockserver.url('/redirect-to'),
|
||||
'goto', url)
|
||||
|
||||
|
||||
|
|
@ -134,7 +134,7 @@ class FileDownloadCrawlTestCase(TestCase):
|
|||
def test_download_media(self):
|
||||
crawler = self._create_crawler(MediaDownloadSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl("http://localhost:8998/files/images/",
|
||||
yield crawler.crawl(self.mockserver.url("/files/images/"),
|
||||
media_key=self.media_key,
|
||||
media_urls_key=self.media_urls_key)
|
||||
self._assert_files_downloaded(self.items, str(log))
|
||||
|
|
@ -143,7 +143,7 @@ class FileDownloadCrawlTestCase(TestCase):
|
|||
def test_download_media_wrong_urls(self):
|
||||
crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl("http://localhost:8998/files/images/",
|
||||
yield crawler.crawl(self.mockserver.url("/files/images/"),
|
||||
media_key=self.media_key,
|
||||
media_urls_key=self.media_urls_key)
|
||||
self._assert_files_download_failure(crawler, self.items, 404, str(log))
|
||||
|
|
@ -152,9 +152,10 @@ class FileDownloadCrawlTestCase(TestCase):
|
|||
def test_download_media_redirected_default_failure(self):
|
||||
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl("http://localhost:8998/files/images/",
|
||||
yield crawler.crawl(self.mockserver.url("/files/images/"),
|
||||
media_key=self.media_key,
|
||||
media_urls_key=self.media_urls_key)
|
||||
media_urls_key=self.media_urls_key,
|
||||
mockserver=self.mockserver)
|
||||
self._assert_files_download_failure(crawler, self.items, 302, str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
|
|
@ -165,9 +166,10 @@ class FileDownloadCrawlTestCase(TestCase):
|
|||
|
||||
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl("http://localhost:8998/files/images/",
|
||||
yield crawler.crawl(self.mockserver.url("/files/images/"),
|
||||
media_key=self.media_key,
|
||||
media_urls_key=self.media_urls_key)
|
||||
media_urls_key=self.media_urls_key,
|
||||
mockserver=self.mockserver)
|
||||
self._assert_files_downloaded(self.items, str(log))
|
||||
self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import json
|
|||
import os
|
||||
import time
|
||||
|
||||
from six.moves.urllib.parse import urlsplit, urlunsplit
|
||||
from threading import Thread
|
||||
from libmproxy import controller, proxy
|
||||
from netlib import http_auth
|
||||
|
|
@ -17,7 +18,7 @@ from tests.mockserver import MockServer
|
|||
|
||||
class HTTPSProxy(controller.Master, Thread):
|
||||
|
||||
def __init__(self, port):
|
||||
def __init__(self):
|
||||
password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
|
||||
authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
|
||||
cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
|
||||
|
|
@ -25,10 +26,19 @@ class HTTPSProxy(controller.Master, Thread):
|
|||
server = proxy.ProxyServer(proxy.ProxyConfig(
|
||||
authenticator = authenticator,
|
||||
cacert = cert_path),
|
||||
port)
|
||||
0)
|
||||
self.server = server
|
||||
Thread.__init__(self)
|
||||
controller.Master.__init__(self, server)
|
||||
|
||||
def http_address(self):
|
||||
return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
|
||||
|
||||
|
||||
def _wrong_credentials(proxy_url):
|
||||
bad_auth_proxy = list(urlsplit(proxy_url))
|
||||
bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
|
||||
return urlunsplit(bad_auth_proxy)
|
||||
|
||||
class ProxyConnectTestCase(TestCase):
|
||||
|
||||
|
|
@ -36,12 +46,14 @@ class ProxyConnectTestCase(TestCase):
|
|||
self.mockserver = MockServer()
|
||||
self.mockserver.__enter__()
|
||||
self._oldenv = os.environ.copy()
|
||||
self._proxy = HTTPSProxy(8888)
|
||||
|
||||
self._proxy = HTTPSProxy()
|
||||
self._proxy.start()
|
||||
|
||||
# Wait for the proxy to start.
|
||||
time.sleep(1.0)
|
||||
os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888'
|
||||
os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
|
||||
os.environ['https_proxy'] = self._proxy.http_address()
|
||||
os.environ['http_proxy'] = self._proxy.http_address()
|
||||
|
||||
def tearDown(self):
|
||||
self.mockserver.__exit__(None, None, None)
|
||||
|
|
@ -52,17 +64,17 @@ class ProxyConnectTestCase(TestCase):
|
|||
def test_https_connect_tunnel(self):
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("https://localhost:8999/status?n=200")
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
|
||||
self._assert_got_response_code(200, l)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_https_noconnect(self):
|
||||
os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect'
|
||||
proxy = os.environ['https_proxy']
|
||||
os.environ['https_proxy'] = proxy + '?noconnect'
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("https://localhost:8999/status?n=200")
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
|
||||
self._assert_got_response_code(200, l)
|
||||
os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_https_connect_tunnel_error(self):
|
||||
|
|
@ -73,18 +85,17 @@ class ProxyConnectTestCase(TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_https_tunnel_auth_error(self):
|
||||
os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888'
|
||||
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("https://localhost:8999/status?n=200")
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
|
||||
# The proxy returns a 407 error code but it does not reach the client;
|
||||
# he just sees a TunnelError.
|
||||
self._assert_got_tunnel_error(l)
|
||||
os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888'
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_https_tunnel_without_leak_proxy_authorization_header(self):
|
||||
request = Request("https://localhost:8999/echo")
|
||||
request = Request(self.mockserver.url("/echo", is_secure=True))
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl(seed=request)
|
||||
|
|
@ -94,10 +105,10 @@ class ProxyConnectTestCase(TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_https_noconnect_auth_error(self):
|
||||
os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect'
|
||||
os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
|
||||
crawler = get_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("https://localhost:8999/status?n=200")
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
|
||||
self._assert_got_response_code(407, l)
|
||||
|
||||
def _assert_got_response_code(self, code, log):
|
||||
|
|
|
|||
|
|
@ -11,20 +11,21 @@ from scrapy.http import Response, Request
|
|||
from scrapy.spiders import Spider
|
||||
from scrapy.spidermiddlewares.httperror import HttpErrorMiddleware, HttpError
|
||||
from scrapy.settings import Settings
|
||||
from tests.spiders import MockServerSpider
|
||||
|
||||
|
||||
class _HttpErrorSpider(Spider):
|
||||
class _HttpErrorSpider(MockServerSpider):
|
||||
name = 'httperror'
|
||||
start_urls = [
|
||||
"http://localhost:8998/status?n=200",
|
||||
"http://localhost:8998/status?n=404",
|
||||
"http://localhost:8998/status?n=402",
|
||||
"http://localhost:8998/status?n=500",
|
||||
]
|
||||
bypass_status_codes = set()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(_HttpErrorSpider, self).__init__(*args, **kwargs)
|
||||
self.start_urls = [
|
||||
self.mockserver.url("/status?n=200"),
|
||||
self.mockserver.url("/status?n=404"),
|
||||
self.mockserver.url("/status?n=402"),
|
||||
self.mockserver.url("/status?n=500"),
|
||||
]
|
||||
self.failed = set()
|
||||
self.skipped = set()
|
||||
self.parsed = set()
|
||||
|
|
@ -169,7 +170,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_middleware_works(self):
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
yield crawler.crawl()
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert not crawler.spider.skipped, crawler.spider.skipped
|
||||
self.assertEqual(crawler.spider.parsed, {'200'})
|
||||
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
|
||||
|
|
@ -184,7 +185,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
def test_logging(self):
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(bypass_status_codes={402})
|
||||
yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
|
||||
self.assertEqual(crawler.spider.parsed, {'200', '402'})
|
||||
self.assertEqual(crawler.spider.skipped, {'402'})
|
||||
self.assertEqual(crawler.spider.failed, {'404', '500'})
|
||||
|
|
@ -199,7 +200,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
# HttpError logs ignored responses with level INFO
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.INFO) as log:
|
||||
yield crawler.crawl()
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.parsed, {'200'})
|
||||
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
|
||||
|
||||
|
|
@ -211,7 +212,7 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
# with level WARNING, we shouldn't capture anything from HttpError
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
with LogCapture(level=logging.WARNING) as log:
|
||||
yield crawler.crawl()
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(crawler.spider.parsed, {'200'})
|
||||
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue