From 04b2f79e7a1316d29b2cb94c8fb2623e040b64ec Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 22:30:58 +0500 Subject: [PATCH] Remove code required to support ancient twisted versions. See GH-1887. --- scrapy/core/downloader/handlers/http.py | 8 +--- tests/mockserver.py | 39 +++++-------------- tests/test_downloader_handlers.py | 49 +++++++++--------------- tests/test_downloadermiddleware_retry.py | 5 +-- 4 files changed, 30 insertions(+), 71 deletions(-) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 81da2615a..e4a7d8564 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,10 +1,6 @@ -from scrapy import twisted_version +from __future__ import absolute_import from .http10 import HTTP10DownloadHandler - -if twisted_version >= (11, 1, 0): - from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler -else: - HTTPDownloadHandler = HTTP10DownloadHandler +from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler # backwards compatibility diff --git a/tests/mockserver.py b/tests/mockserver.py index a40e2e501..e611cc3ec 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -2,34 +2,18 @@ from __future__ import print_function import sys, time, random, os, json from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE + from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource -from twisted.internet import reactor, defer, ssl -from scrapy import twisted_version +from twisted.web.test.test_webclient import PayloadResource +from twisted.web.server import GzipEncoderFactory +from twisted.web.resource import EncodingResourceWrapper +from twisted.internet import reactor, ssl +from twisted.internet.task import deferLater + from scrapy.utils.python import to_bytes, to_unicode -if twisted_version < (11, 0, 0): - def deferLater(clock, delay, func, *args, **kw): - def _cancel_method(): - _cancel_cb(None) - d.errback(Exception()) - - def _cancel_cb(result): - if cl.active(): - cl.cancel() - return result - - d = defer.Deferred() - d.cancel = _cancel_method - d.addCallback(lambda ignored: func(*args, **kw)) - d.addBoth(_cancel_cb) - cl = clock.callLater(delay, d.callback, None) - return d -else: - from twisted.internet.task import deferLater - - def getarg(request, name, default=None, type=None): if name in request.args: value = request.args[name][0] @@ -174,13 +158,8 @@ class Root(Resource): self.putChild(b"drop", Drop()) self.putChild(b"raw", Raw()) self.putChild(b"echo", Echo()) - - if twisted_version > (12, 3, 0): - from twisted.web.test.test_webclient import PayloadResource - from twisted.web.server import GzipEncoderFactory - from twisted.web.resource import EncodingResourceWrapper - self.putChild(b"payload", PayloadResource()) - self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) + self.putChild(b"payload", PayloadResource()) + self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()])) def getChild(self, name, request): return self diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8d3b49d6a..6333efceb 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -17,7 +17,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ from twisted.cred import portal, checkers, credentials from w3lib.url import path_to_file_uri -from scrapy import twisted_version from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler @@ -281,8 +280,6 @@ class Https10TestCase(Http10TestCase): class Http11TestCase(HttpTestCase): """HTTP 1.1 test case""" download_handler_cls = HTTP11DownloadHandler - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' def test_download_without_maxsize_limit(self): request = Request(self.getURL('file')) @@ -366,8 +363,6 @@ class Https11InvalidDNSId(Https11TestCase): class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' def setUp(self): self.mockserver = MockServer() @@ -396,31 +391,27 @@ class Http11MockServerTestCase(unittest.TestCase): @defer.inlineCallbacks def test_download_gzip_response(self): + crawler = get_crawler(SingleRequestSpider) + body = b'1' * 100 # PayloadResource requires body length to be 100 + request = Request('http://localhost:8998/payload', method='POST', + body=body, meta={'download_maxsize': 50}) + yield crawler.crawl(seed=request) + failure = crawler.spider.meta['failure'] + # download_maxsize < 100, hence the CancelledError + self.assertIsInstance(failure.value, defer.CancelledError) - if twisted_version > (12, 3, 0): - - crawler = get_crawler(SingleRequestSpider) - body = b'1'*100 # PayloadResource requires body length to be 100 - request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50}) + if six.PY2: + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url='http://localhost:8998/xpayload') yield crawler.crawl(seed=request) - failure = crawler.spider.meta['failure'] - # download_maxsize < 100, hence the CancelledError - self.assertIsInstance(failure.value, defer.CancelledError) - - if six.PY2: - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url='http://localhost:8998/xpayload') - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure == None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') - else: - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure == None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') else: - raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0") + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload only enabled for PY2") class UriResource(resource.Resource): @@ -500,8 +491,6 @@ class Http10ProxyTestCase(HttpProxyTestCase): class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler - if twisted_version < (11, 1, 0): - skip = 'HTTP1.1 not supported in twisted < 11.1.0' @defer.inlineCallbacks def test_download_with_proxy_https_timeout(self): @@ -692,8 +681,6 @@ class FTPTestCase(unittest.TestCase): username = "scrapy" password = "passwd" - if twisted_version < (10, 2, 0): - skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" if six.PY3: skip = "Twisted missing ftp support for PY3" diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index eb17974bf..e129b71f8 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \ ConnectionLost, TCPTimedOutError from twisted.web.client import ResponseFailed -from scrapy import twisted_version from scrapy.downloadermiddlewares.retry import RetryMiddleware from scrapy.spiders import Spider from scrapy.http import Request, Response @@ -74,9 +73,7 @@ class RetryTest(unittest.TestCase): def test_twistederrors(self): exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, - ConnectError, ConnectionLost] - if twisted_version >= (11, 1, 0): # http11 available - exceptions.append(ResponseFailed) + ConnectError, ConnectionLost, ResponseFailed] for exc in exceptions: req = Request('http://www.scrapytest.org/%s' % exc.__name__)