remove ssl from optional_features to simplify code, as it is now required. also deprecate optional_features set

scrapy/__init__.py

@@ -17,15 +17,11 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 # monkey patches to fix external library issues
 from scrapy.xlib import urlparse_monkeypatches
 
-# optional_features is a set containing Scrapy optional features
+# WARNING: optional_features set is deprecated and will be removed soon. Do not use.
 optional_features = set()
 
-try:
-    import OpenSSL
-except ImportError:
-    pass
-else:
-    optional_features.add('ssl')
+# TODO: backwards compatibility, remove for Scrapy 0.20
+optional_features.add('ssl')
 
 try:
     import boto

scrapy/core/downloader/handlers/http.py

@@ -2,11 +2,7 @@
 
 from twisted.internet import reactor
 
-from scrapy.exceptions import NotSupported
 from scrapy.utils.misc import load_object
-from scrapy import optional_features
-
-ssl_supported = 'ssl' in optional_features
 
 class HttpDownloadHandler(object):
 
@@ -23,9 +19,7 @@ class HttpDownloadHandler(object):
     def _connect(self, factory):
         host, port = factory.host, factory.port
         if factory.scheme == 'https':
-            if ssl_supported:
-                return reactor.connectSSL(host, port, factory, \
-                        self.ClientContextFactory())
-            raise NotSupported("HTTPS not supported: install pyopenssl library")
+            return reactor.connectSSL(host, port, factory, \
+                    self.ClientContextFactory())
         else:
             return reactor.connectTCP(host, port, factory)

scrapy/core/downloader/webclient.py

@@ -1,14 +1,15 @@
 from time import time
 from urlparse import urlparse, urlunparse, urldefrag
 
+from twisted.internet.ssl import ClientContextFactory
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient
 from twisted.internet import defer
+from OpenSSL import SSL
 
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.responsetypes import responsetypes
-from scrapy import optional_features
 
 
 def _parsed_url_args(parsed):
@@ -138,13 +139,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
 
 
 
-if 'ssl' in optional_features:
-    from twisted.internet.ssl import ClientContextFactory
-    from OpenSSL import SSL
-else:
-    ClientContextFactory = object
-
-
 class ScrapyClientContextFactory(ClientContextFactory):
     "A SSL context factory which is more permissive against SSL bugs."
     # see https://github.com/scrapy/scrapy/issues/82