From 3d4317bfe4697955de1c2450ce4cb0d12471a9a3 Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 21 Oct 2019 18:32:30 +0100 Subject: [PATCH 01/20] [tox.ini] Added python 3.8 fields https://github.com/scrapy/scrapy/issues/4085 --- tox.ini | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tox.ini b/tox.ini index ffe7360d3..14afec23f 100644 --- a/tox.ini +++ b/tox.ini @@ -92,6 +92,10 @@ deps = {[testenv:py35]deps} basepython = python3.7 deps = {[testenv:py35]deps} +[testenv:py38] +basepython = python3.8 +deps = {[testenv:py35]deps} + [testenv:pypy3] basepython = pypy3 deps = {[testenv:py35]deps} @@ -128,6 +132,13 @@ deps = reppy robotexclusionrulesparser +[testenv:py38-extra-deps] +basepython = python3.8 +deps = + {[testenv:py35]deps} + reppy + robotexclusionrulesparser + [testenv:py27-extra-deps] basepython = python2.7 deps = From 4e939ca75d14daf5bc658fdbe5a97af6f4c3c498 Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 21 Oct 2019 18:33:18 +0100 Subject: [PATCH 02/20] [setup.py] Added python 3.8 fields https://github.com/scrapy/scrapy/issues/4085 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 850456503..2f5fca4c9 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From c12a075164e95e98a136499faf7bf8eefdce0300 Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 21 Oct 2019 18:34:15 +0100 Subject: [PATCH 03/20] [.travis.yml] Added python 3.8 fields https://github.com/scrapy/scrapy/issues/4085 --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0190a7f4d..2ba504972 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,10 @@ matrix: python: 3.7 - env: TOXENV=py37-extra-deps python: 3.7 + - env: TOXENV=py38 + python: 3.8 + - env: TOXENV=py37-extra-deps + python: 3.8 - env: TOXENV=docs python: 3.6 install: From 85ac5c5c5778a116f762cb6200492aa6a2439e83 Mon Sep 17 00:00:00 2001 From: Roy Healy Date: Mon, 21 Oct 2019 19:06:43 +0100 Subject: [PATCH 04/20] Update .travis.yml Co-Authored-By: Mikhail Korobov --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ba504972..044fa9e95 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ matrix: python: 3.7 - env: TOXENV=py38 python: 3.8 - - env: TOXENV=py37-extra-deps + - env: TOXENV=py38-extra-deps python: 3.8 - env: TOXENV=docs python: 3.6 From 179dc916ff3e3518d8874f3fcd93948e11d5a259 Mon Sep 17 00:00:00 2001 From: Roy Healy Date: Sun, 27 Oct 2019 16:48:20 +0000 Subject: [PATCH 05/20] Update setup.py Updating setup.py to remove python 3.8 support for now --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2f5fca4c9..4127d3191 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + # 'Programming Language :: Python :: 3.8', not supported yet 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 4068797558f6d8d58f81e12930f487fc0f59f0a5 Mon Sep 17 00:00:00 2001 From: Roy Healy Date: Sun, 27 Oct 2019 17:02:17 +0000 Subject: [PATCH 06/20] Update test_downloadermiddleware_httpcache.py Adding xfail denoting that leveldb is not supported in 3.8 --- tests/test_downloadermiddleware_httpcache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 22946b98c..34bf5776a 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -154,6 +154,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): new_settings.setdefault('HTTPCACHE_GZIP', True) return super(FilesystemStorageTest, self)._get_settings(**new_settings) +@pytest.mark.xfail(reason='leveldb not supported in python 3.8') class LeveldbStorageTest(DefaultStorageTest): pytest.importorskip('leveldb') From deacd34c8d2400b07e911dea9173b840cec7dece Mon Sep 17 00:00:00 2001 From: Roy Date: Sun, 27 Oct 2019 17:39:47 +0000 Subject: [PATCH 07/20] [test_downloadermiddleware_httpcache] Attempting to add xfail for leveldb related tests https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 34bf5776a..e350da72c 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -104,6 +104,7 @@ class _BaseTest(unittest.TestCase): class DefaultStorageTest(_BaseTest): + @pytest.mark.xfail(reason='leveldb not supported in python 3.8') def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -154,7 +155,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): new_settings.setdefault('HTTPCACHE_GZIP', True) return super(FilesystemStorageTest, self)._get_settings(**new_settings) -@pytest.mark.xfail(reason='leveldb not supported in python 3.8') class LeveldbStorageTest(DefaultStorageTest): pytest.importorskip('leveldb') From 11942c436c78e998e3d58f0aad7cba490d74cc67 Mon Sep 17 00:00:00 2001 From: Roy Date: Sun, 27 Oct 2019 18:02:13 +0000 Subject: [PATCH 08/20] [test_downloadermiddleware_httpcache] Trying hack to handle systemerror whjen importing leveldb https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index e350da72c..eec0feafc 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -6,6 +6,7 @@ import unittest import email.utils from contextlib import contextmanager import pytest +import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider @@ -157,7 +158,10 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): class LeveldbStorageTest(DefaultStorageTest): - pytest.importorskip('leveldb') + try: + pytest.importorskip('leveldb') + except SystemError: + pass storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage' From b3df0a84150f15ff00cdfaffc586f91b653baecd Mon Sep 17 00:00:00 2001 From: Roy Date: Sun, 27 Oct 2019 18:28:47 +0000 Subject: [PATCH 09/20] [test_downloadermiddleware_httpcache] Adding xfails to impacted tests following hack fix https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index eec0feafc..605223088 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -6,7 +6,6 @@ import unittest import email.utils from contextlib import contextmanager import pytest -import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider @@ -90,6 +89,7 @@ class _BaseTest(unittest.TestCase): assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) self.assertEqual(request1.body, request2.body) + @pytest.mark.xfail(reason='leveldb not supported in python 3.8') def test_dont_cache(self): with self._middleware() as mw: self.request.meta['dont_cache'] = True @@ -119,6 +119,7 @@ class DefaultStorageTest(_BaseTest): time.sleep(2) # wait for cache to expire assert storage.retrieve_response(self.spider, request2) is None + @pytest.mark.xfail(reason='leveldb not supported in python 3.8') def test_storage_never_expire(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage: assert storage.retrieve_response(self.spider, self.request) is None @@ -161,6 +162,8 @@ class LeveldbStorageTest(DefaultStorageTest): try: pytest.importorskip('leveldb') except SystemError: + # Happens in python 3.8 + # This will cause xfail in DefaultStorageTest to trigger pass storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage' From 70b2854590c1aa10324e6a4b50b6d74a5079c8e9 Mon Sep 17 00:00:00 2001 From: Roy Date: Sun, 27 Oct 2019 18:51:26 +0000 Subject: [PATCH 10/20] [test_downloadermiddleware_httpcache] Making xfails more informative https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 605223088..faee37446 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -6,6 +6,7 @@ import unittest import email.utils from contextlib import contextmanager import pytest +import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider @@ -89,7 +90,7 @@ class _BaseTest(unittest.TestCase): assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) self.assertEqual(request1.body, request2.body) - @pytest.mark.xfail(reason='leveldb not supported in python 3.8') + @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8') def test_dont_cache(self): with self._middleware() as mw: self.request.meta['dont_cache'] = True @@ -105,7 +106,7 @@ class _BaseTest(unittest.TestCase): class DefaultStorageTest(_BaseTest): - @pytest.mark.xfail(reason='leveldb not supported in python 3.8') + @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8') def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -119,7 +120,7 @@ class DefaultStorageTest(_BaseTest): time.sleep(2) # wait for cache to expire assert storage.retrieve_response(self.spider, request2) is None - @pytest.mark.xfail(reason='leveldb not supported in python 3.8') + @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8') def test_storage_never_expire(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage: assert storage.retrieve_response(self.spider, self.request) is None From 20ea912513053aa2b2122ba5bb0387189bbc5467 Mon Sep 17 00:00:00 2001 From: Roy Date: Sun, 27 Oct 2019 18:52:01 +0000 Subject: [PATCH 11/20] [test_downloadermiddleware_httpcache] Making xfails more informative https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index faee37446..b832fc38c 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -90,7 +90,7 @@ class _BaseTest(unittest.TestCase): assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) self.assertEqual(request1.body, request2.body) - @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8') + @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8') def test_dont_cache(self): with self._middleware() as mw: self.request.meta['dont_cache'] = True @@ -106,7 +106,7 @@ class _BaseTest(unittest.TestCase): class DefaultStorageTest(_BaseTest): - @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8') + @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8') def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -120,7 +120,7 @@ class DefaultStorageTest(_BaseTest): time.sleep(2) # wait for cache to expire assert storage.retrieve_response(self.spider, request2) is None - @pytest.mark.xfail(sys.version_info >= (3, 8), raises=RuntimeError, reason='leveldb not supported in python 3.8') + @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8') def test_storage_never_expire(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage: assert storage.retrieve_response(self.spider, self.request) is None From b5a00262ec48534b89750037060318326b4e349c Mon Sep 17 00:00:00 2001 From: Roy Healy Date: Mon, 28 Oct 2019 09:59:33 +0000 Subject: [PATCH 12/20] Update .travis.yml Reverting change to 3.8 extra dependency environment. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 044fa9e95..2ba504972 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ matrix: python: 3.7 - env: TOXENV=py38 python: 3.8 - - env: TOXENV=py38-extra-deps + - env: TOXENV=py37-extra-deps python: 3.8 - env: TOXENV=docs python: 3.6 From 3d0df419c4eeb0ccf7934c8f06a38707eae8d722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 28 Oct 2019 11:24:47 +0100 Subject: [PATCH 13/20] Mark the LevelDB storage backend as deprecated --- docs/topics/downloader-middleware.rst | 22 ---------------------- scrapy/extensions/httpcache.py | 17 ++++++++++++----- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 2892b9b79..8048e1c86 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -348,7 +348,6 @@ HttpCacheMiddleware * :ref:`httpcache-storage-fs` * :ref:`httpcache-storage-dbm` - * :ref:`httpcache-storage-leveldb` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also :ref:`implement your own storage backend. ` @@ -478,27 +477,6 @@ DBM storage backend By default, it uses the anydbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. -.. _httpcache-storage-leveldb: - -LevelDB storage backend -~~~~~~~~~~~~~~~~~~~~~~~ - -.. class:: LeveldbCacheStorage - - .. versionadded:: 0.23 - - A LevelDB_ storage backend is also available for the HTTP cache middleware. - - This backend is not recommended for development because only one process - can access LevelDB databases at the same time, so you can't run a crawl and - open the scrapy shell in parallel for the same spider. - - In order to use this storage backend, install the `LevelDB python - bindings`_ (e.g. ``pip install leveldb``). - - .. _LevelDB: https://github.com/google/leveldb - .. _leveldb python bindings: https://pypi.python.org/pypi/leveldb - .. _httpcache-storage-custom: Writing your own storage backend diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index c6094643d..7c650a91e 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,19 +1,24 @@ from __future__ import print_function -import os + import gzip import logging -from six.moves import cPickle as pickle +import os +from email.utils import mktime_tz, parsedate_tz from importlib import import_module from time import time +from warnings import warn from weakref import WeakKeyDictionary -from email.utils import mktime_tz, parsedate_tz + +from six.moves import cPickle as pickle from w3lib.http import headers_raw_to_dict, headers_dict_to_raw + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes -from scrapy.utils.request import request_fingerprint -from scrapy.utils.project import data_path from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode, garbage_collect +from scrapy.utils.request import request_fingerprint logger = logging.getLogger(__name__) @@ -345,6 +350,8 @@ class FilesystemCacheStorage(object): class LeveldbCacheStorage(object): def __init__(self, settings): + warn("The LevelDB storage backend is deprecated.", + ScrapyDeprecationWarning, stacklevel=2) import leveldb self._leveldb = leveldb self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) From 16bb3ac20dae8b7c5fbccf4ab85b3a0393e7c55d Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 28 Oct 2019 11:24:09 +0000 Subject: [PATCH 14/20] [test_downloadermiddleware_httpcache] Using skipif approach https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index b832fc38c..ba5027307 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -90,7 +90,6 @@ class _BaseTest(unittest.TestCase): assert any(h in request2.headers for h in (b'If-None-Match', b'If-Modified-Since')) self.assertEqual(request1.body, request2.body) - @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8') def test_dont_cache(self): with self._middleware() as mw: self.request.meta['dont_cache'] = True @@ -106,7 +105,6 @@ class _BaseTest(unittest.TestCase): class DefaultStorageTest(_BaseTest): - @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8') def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -120,7 +118,6 @@ class DefaultStorageTest(_BaseTest): time.sleep(2) # wait for cache to expire assert storage.retrieve_response(self.spider, request2) is None - @pytest.mark.xfail(sys.version_info >= (3, 8), raises=SystemError, reason='leveldb not supported in python 3.8') def test_storage_never_expire(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=0) as storage: assert storage.retrieve_response(self.spider, self.request) is None @@ -164,8 +161,10 @@ class LeveldbStorageTest(DefaultStorageTest): pytest.importorskip('leveldb') except SystemError: # Happens in python 3.8 - # This will cause xfail in DefaultStorageTest to trigger - pass + pytest.mark.skipif( + sys.version_info >= (3, 8), + reason='leveldb not supported in python 3.8', + ) storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage' From 9b47dc6a703310d13c9470e50d4b14f81ee893c6 Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 28 Oct 2019 11:24:52 +0000 Subject: [PATCH 15/20] [travis, setup] Adding official python 3.8 support https://github.com/scrapy/scrapy/issues/4085 --- .travis.yml | 4 +--- setup.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ba504972..4c2498053 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,11 +25,9 @@ matrix: python: 3.6 - env: TOXENV=py37 python: 3.7 - - env: TOXENV=py37-extra-deps - python: 3.7 - env: TOXENV=py38 python: 3.8 - - env: TOXENV=py37-extra-deps + - env: TOXENV=py38-extra-deps python: 3.8 - env: TOXENV=docs python: 3.6 diff --git a/setup.py b/setup.py index 4127d3191..2f5fca4c9 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', - # 'Programming Language :: Python :: 3.8', not supported yet + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', From 4432136ffff4d8af42f7a485c17ab7fbbb228078 Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 28 Oct 2019 12:22:21 +0000 Subject: [PATCH 16/20] [test_downloadermiddleware_httpcache] Fixing pytest skip behaviour https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index ba5027307..32085b095 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -6,7 +6,6 @@ import unittest import email.utils from contextlib import contextmanager import pytest -import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider @@ -161,10 +160,7 @@ class LeveldbStorageTest(DefaultStorageTest): pytest.importorskip('leveldb') except SystemError: # Happens in python 3.8 - pytest.mark.skipif( - sys.version_info >= (3, 8), - reason='leveldb not supported in python 3.8', - ) + pytest.skip("'SystemError: bad call flags' occurs on Python 3.8") storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage' From c51fb959e2985faf6f21fe7f03d2fb8160de064f Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 28 Oct 2019 12:36:54 +0000 Subject: [PATCH 17/20] [test_downloadermiddleware_httpcache] Fixing pytest skip behaviour https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 32085b095..047526569 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -6,6 +6,7 @@ import unittest import email.utils from contextlib import contextmanager import pytest +import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider @@ -160,7 +161,7 @@ class LeveldbStorageTest(DefaultStorageTest): pytest.importorskip('leveldb') except SystemError: # Happens in python 3.8 - pytest.skip("'SystemError: bad call flags' occurs on Python 3.8") + pytestmark = pytest.skip("'SystemError: bad call flags' occurs on Python 3.8") storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage' From 74909030a55b59e3b858fc736b5b1f685d9596a6 Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 28 Oct 2019 12:52:36 +0000 Subject: [PATCH 18/20] [tox.ini] Removing obsolete py37 extra deps enviornment https://github.com/scrapy/scrapy/issues/4085 --- tox.ini | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tox.ini b/tox.ini index 14afec23f..fe925951b 100644 --- a/tox.ini +++ b/tox.ini @@ -125,13 +125,6 @@ deps = {[docs]deps} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck -[testenv:py37-extra-deps] -basepython = python3.7 -deps = - {[testenv:py35]deps} - reppy - robotexclusionrulesparser - [testenv:py38-extra-deps] basepython = python3.8 deps = From b73d217de5647a68c7b8dfda747cd3d0685c226d Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 28 Oct 2019 12:55:54 +0000 Subject: [PATCH 19/20] [test_downloadermiddleware_httpcache.py] Fixing pytest mark behaviour https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 047526569..f5917d0f0 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -161,7 +161,7 @@ class LeveldbStorageTest(DefaultStorageTest): pytest.importorskip('leveldb') except SystemError: # Happens in python 3.8 - pytestmark = pytest.skip("'SystemError: bad call flags' occurs on Python 3.8") + pytestmark = pytest.mark.skip("'SystemError: bad call flags' occurs on Python 3.8") storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage' From 93e3dc1b826e44d1a5a24fbb39c090ce426aa862 Mon Sep 17 00:00:00 2001 From: Roy Date: Mon, 28 Oct 2019 16:12:03 +0000 Subject: [PATCH 20/20] [test_downloadermiddleware_httpcache.py] Cleaning text https://github.com/scrapy/scrapy/issues/4085 --- tests/test_downloadermiddleware_httpcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index f5917d0f0..972d400a4 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -155,13 +155,13 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): new_settings.setdefault('HTTPCACHE_GZIP', True) return super(FilesystemStorageTest, self)._get_settings(**new_settings) + class LeveldbStorageTest(DefaultStorageTest): try: pytest.importorskip('leveldb') except SystemError: - # Happens in python 3.8 - pytestmark = pytest.mark.skip("'SystemError: bad call flags' occurs on Python 3.8") + pytestmark = pytest.mark.skip("Test module skipped - 'SystemError: bad call flags' occurs when >= Python 3.8") storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'