Merge pull request #4092 from further-reading/master

Add Python 3.8 official support

.travis.yml

@@ -25,8 +25,10 @@ matrix:
       python: 3.6
     - env: TOXENV=py37
       python: 3.7
-    - env: TOXENV=py37-extra-deps
-      python: 3.7
+    - env: TOXENV=py38
+      python: 3.8
+    - env: TOXENV=py38-extra-deps
+      python: 3.8
     - env: TOXENV=docs
       python: 3.6
 install:

docs/topics/downloader-middleware.rst

@@ -348,7 +348,6 @@ HttpCacheMiddleware
 
         * :ref:`httpcache-storage-fs`
         * :ref:`httpcache-storage-dbm`
-        * :ref:`httpcache-storage-leveldb`
 
     You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
     setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
@@ -478,27 +477,6 @@ DBM storage backend
     By default, it uses the anydbm_ module, but you can change it with the
     :setting:`HTTPCACHE_DBM_MODULE` setting.
 
-.. _httpcache-storage-leveldb:
-
-LevelDB storage backend
-~~~~~~~~~~~~~~~~~~~~~~~
-
-.. class:: LeveldbCacheStorage
-
-    .. versionadded:: 0.23
-
-    A LevelDB_ storage backend is also available for the HTTP cache middleware.
-
-    This backend is not recommended for development because only one process
-    can access LevelDB databases at the same time, so you can't run a crawl and
-    open the scrapy shell in parallel for the same spider.
-
-    In order to use this storage backend, install the `LevelDB python
-    bindings`_ (e.g. ``pip install leveldb``).
-
-    .. _LevelDB: https://github.com/google/leveldb
-    .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
-
 .. _httpcache-storage-custom:
 
 Writing your own storage backend

scrapy/extensions/httpcache.py

@@ -1,19 +1,24 @@
 from __future__ import print_function
-import os
+
 import gzip
 import logging
-from six.moves import cPickle as pickle
+import os
+from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from time import time
+from warnings import warn
 from weakref import WeakKeyDictionary
-from email.utils import mktime_tz, parsedate_tz
+
+from six.moves import cPickle as pickle
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
-from scrapy.utils.request import request_fingerprint
-from scrapy.utils.project import data_path
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
+from scrapy.utils.request import request_fingerprint
 
 
 logger = logging.getLogger(__name__)
@@ -345,6 +350,8 @@ class FilesystemCacheStorage(object):
 class LeveldbCacheStorage(object):
 
     def __init__(self, settings):
+        warn("The LevelDB storage backend is deprecated.",
+             ScrapyDeprecationWarning, stacklevel=2)
         import leveldb
         self._leveldb = leveldb
         self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)

setup.py

@@ -56,6 +56,7 @@ setup(
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',

tests/test_downloadermiddleware_httpcache.py

@@ -6,6 +6,7 @@ import unittest
 import email.utils
 from contextlib import contextmanager
 import pytest
+import sys
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spiders import Spider
@@ -154,9 +155,13 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
         new_settings.setdefault('HTTPCACHE_GZIP', True)
         return super(FilesystemStorageTest, self)._get_settings(**new_settings)
 
+
 class LeveldbStorageTest(DefaultStorageTest):
 
-    pytest.importorskip('leveldb')
+    try:
+        pytest.importorskip('leveldb')
+    except SystemError:
+        pytestmark = pytest.mark.skip("Test module skipped - 'SystemError: bad call flags' occurs when >= Python 3.8")
     storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
 
 

tox.ini

@@ -92,6 +92,10 @@ deps = {[testenv:py35]deps}
 basepython = python3.7
 deps = {[testenv:py35]deps}
 
+[testenv:py38]
+basepython = python3.8
+deps = {[testenv:py35]deps}
+
 [testenv:pypy3]
 basepython = pypy3
 deps = {[testenv:py35]deps}
@@ -121,8 +125,8 @@ deps = {[docs]deps}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
 
-[testenv:py37-extra-deps]
-basepython = python3.7
+[testenv:py38-extra-deps]
+basepython = python3.8
 deps =
     {[testenv:py35]deps}
     reppy