From a69f042d1064c4beea46fa70084835dd6c91c143 Mon Sep 17 00:00:00 2001 From: tpeng Date: Wed, 19 Nov 2014 11:50:07 +0100 Subject: [PATCH] add 2 more test cases and minor doc fixes --- docs/topics/settings.rst | 10 ++++++--- scrapy/settings/default_settings.py | 4 ++-- tests/mockserver.py | 7 ++++-- tests/test_downloader_handlers.py | 34 +++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 402226743..0e9e53de4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -427,7 +427,7 @@ The amount of time (in secs) that the downloader will wait before timing out. DOWNLOAD_MAXSIZE ---------------- -Default: `1073741824` (1024Mb) +Default: `1073741824` (1024MB) The maximum response size (in bytes) that downloader will download. @@ -439,12 +439,14 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_maxsize` Request.meta key. + This feature needs Twisted >= 11.1. + .. setting:: DOWNLOAD_WARNSIZE DOWNLOAD_WARNSIZE ----------------- +----------------- -Default: `33554432` (32Mb) +Default: `33554432` (32MB) The response size (in bytes) that downloader will start to warn. @@ -456,6 +458,8 @@ If you want to disable it set to 0. spider attribute and per-request using :reqmeta:`download_warnsize` Request.meta key. + This feature needs Twisted >= 11.1. + .. setting:: DUPEFILTER_CLASS DUPEFILTER_CLASS diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1b7b3bf29..cf216385a 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -66,8 +66,8 @@ DOWNLOAD_HANDLERS_BASE = { DOWNLOAD_TIMEOUT = 180 # 3mins -DOWNLOAD_MAXSIZE = 1073741824 # 1024m -DOWNLOAD_WARNSIZE = 33554432 # 32m +DOWNLOAD_MAXSIZE = 1024*1024*1024 # 1024m +DOWNLOAD_WARNSIZE = 32*1024*1024 # 32m DOWNLOADER = 'scrapy.core.downloader.Downloader' diff --git a/tests/mockserver.py b/tests/mockserver.py index 6910532b6..2c0ad66fb 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,9 +1,10 @@ from __future__ import print_function import sys, time, random, urllib, os, json from subprocess import Popen, PIPE -from twisted.web.server import Site, NOT_DONE_YET -from twisted.web.resource import Resource +from twisted.web.server import Site, NOT_DONE_YET, GzipEncoderFactory +from twisted.web.resource import Resource, EncodingResourceWrapper from twisted.internet import reactor, defer, ssl +from twisted.web.test.test_webclient import PayloadResource from scrapy import twisted_version @@ -167,6 +168,8 @@ class Root(Resource): self.putChild("drop", Drop()) self.putChild("raw", Raw()) self.putChild("echo", Echo()) + self.putChild('payload', PayloadResource()) + self.putChild("xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) def getChild(self, name, request): return self diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 55bb7ccf7..127c4a4bd 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -220,6 +220,20 @@ class Http11TestCase(HttpTestCase): d.addCallback(self.assertEquals, "0123456789") return d + @defer.inlineCallbacks + def test_download_with_maxsize(self): + request = Request(self.getURL('file')) + + # 10 is minimal size for this request and the limit is only counted on + # response body. (regardless of headers) + d = self.download_request(request, Spider('foo', download_maxsize=10)) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, "0123456789") + yield d + + d = self.download_request(request, Spider('foo', download_maxsize=9)) + yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted) + @defer.inlineCallbacks def test_download_with_maxsize_per_req(self): meta = {'download_maxsize': 2} @@ -271,6 +285,26 @@ class Http11MockServerTestCase(unittest.TestCase): reason = crawler.spider.meta['close_reason'] self.assertTrue(reason, 'finished') + @defer.inlineCallbacks + def test_download_gzip_response(self): + crawler = get_crawler(SingleRequestSpider) + body = '1'*100 # PayloadResource requires body length to be 100 + request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) + yield crawler.crawl(seed=request) + failure = crawler.spider.meta['failure'] + # download_maxsize < 100, hence the CancelledError + self.assertIsInstance(failure.value, defer.CancelledError) + + request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request = request.replace(url='http://localhost:8998/xpayload') + yield crawler.crawl(seed=request) + + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') + class UriResource(resource.Resource): """Return the full uri that was requested"""