Update Python version references after dropping support for 3.5 (#4742)

* Update Python version references after dropping support for 3.5

* Remove outdated test

* Undo change affecting collect_asyncgen

* Undo change to be handled by #4743

* Remove unused import

* Remove unused import

* Update tests/requirements-py3.txt

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>

README.rst

@@ -40,7 +40,7 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.5.2+
+* Python 3.6+
 * Works on Linux, Windows, macOS, BSD
 
 Install

docs/intro/install.rst

@@ -9,8 +9,8 @@ Installation guide
 Supported Python versions
 =========================
 
-Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
-the PyPy 5.9+ implementation (see :ref:`python:implementations`).
+Scrapy requires Python 3.6+, either the CPython implementation (default) or
+the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
 
 
 Installing Scrapy

docs/intro/tutorial.rst

@@ -405,8 +405,6 @@ to get all of them:
 
   from sys import version_info
 
-.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
-
 Having figured out how to extract each bit, we can now iterate over all the
 quotes elements and put them together into a Python dictionary:
 

docs/topics/coroutines.rst

@@ -17,19 +17,14 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.http.Request` callbacks.
 
-    The following are known caveats of the current implementation that we aim
-    to address in future versions of Scrapy:
-
-    -   The callback output is not processed until the whole callback finishes.
+    .. note:: The callback output is not processed until the whole callback
+        finishes.
 
         As a side effect, if the callback raises an exception, none of its
         output is processed.
 
-    -   Because `asynchronous generators were introduced in Python 3.6`_, you
-        can only use ``yield`` if you are using Python 3.6 or later.
-
-        If you need to output multiple items or requests and you are using
-        Python 3.5, return an iterable (e.g. a list) instead.
+        This is a known caveat of the current implementation that we aim to
+        address in a future version of Scrapy.
 
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
@@ -44,8 +39,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
-.. _asynchronous generators were introduced in Python 3.6: https://www.python.org/dev/peps/pep-0525/
-
 Usage
 =====
 

scrapy/__init__.py

@@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 5, 2):
-    print("Scrapy %s requires Python 3.5.2" % __version__)
+if sys.version_info < (3, 6):
+    print("Scrapy %s requires Python 3.6+" % __version__)
     sys.exit(1)
 
 

scrapy/utils/defer.py

@@ -124,18 +124,11 @@ def iter_errback(iterable, errback, *a, **kw):
             errback(failure.Failure(), *a, **kw)
 
 
-def _isfuture(o):
-    # workaround for Python before 3.5.3 not having asyncio.isfuture
-    if hasattr(asyncio, 'isfuture'):
-        return asyncio.isfuture(o)
-    return isinstance(o, asyncio.Future)
-
-
 def deferred_from_coro(o):
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, defer.Deferred):
         return o
-    if _isfuture(o) or inspect.isawaitable(o):
+    if asyncio.isfuture(o) or inspect.isawaitable(o):
         if not is_asyncio_reactor_installed():
             # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
             # that use asyncio, e.g. "await asyncio.sleep(1)"
@@ -167,7 +160,7 @@ def maybeDeferred_coro(f, *args, **kw):
 
     if isinstance(result, defer.Deferred):
         return result
-    elif _isfuture(result) or inspect.isawaitable(result):
+    elif asyncio.isfuture(result) or inspect.isawaitable(result):
         return deferred_from_coro(result)
     elif isinstance(result, failure.Failure):
         return defer.fail(result)

scrapy/utils/gz.py

@@ -6,11 +6,10 @@ import struct
 from scrapy.utils.decorators import deprecated
 
 
-# - Python>=3.5 GzipFile's read() has issues returning leftover
-#   uncompressed data when input is corrupted
-#   (regression or bug-fix compared to Python 3.4)
+# - GzipFile's read() has issues returning leftover uncompressed data when
+#   input is corrupted
 # - read1(), which fetches data before raising EOFError on next call
-#   works here but is only available from Python>=3.3
+#   works here
 @deprecated('GzipFile.read1')
 def read1(gzf, size=-1):
     return gzf.read1(size)

setup.py

@@ -82,7 +82,6 @@ setup(
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
@@ -92,7 +91,7 @@ setup(
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.5.2',
+    python_requires='>=3.6',
     install_requires=install_requires,
     extras_require=extras_require,
 )

tests/requirements-py3.txt

@@ -2,8 +2,7 @@
 attrs
 dataclasses; python_version == '3.6'
 mitmproxy; python_version >= '3.7'
-mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7'
-mitmproxy < 4; python_version < '3.6'
+mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7'
 pyftpdlib
 # https://github.com/pytest-dev/pytest-twisted/issues/93
 pytest != 5.4, != 5.4.1

tests/test_crawl.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import sys
 from ipaddress import IPv4Address
 from socket import gethostbyname
 from urllib.parse import urlparse
@@ -405,7 +404,6 @@ class CrawlSpiderTestCase(TestCase):
         self.assertIn("Got response 200", str(log))
         self.assertIn({"foo": 42}, items)
 
-    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse(self):
@@ -417,7 +415,6 @@ class CrawlSpiderTestCase(TestCase):
         itemcount = crawler.stats.get_value('item_scraped_count')
         self.assertEqual(itemcount, 1)
 
-    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_loop(self):
@@ -437,7 +434,6 @@ class CrawlSpiderTestCase(TestCase):
         for i in range(10):
             self.assertIn({'foo': i}, items)
 
-    @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_complex(self):

tests/test_item.py

@@ -1,4 +1,3 @@
-import sys
 import unittest
 from unittest import mock
 from warnings import catch_warnings
@@ -7,9 +6,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import ABCMeta, _BaseItem, BaseItem, DictItem, Field, Item, ItemMeta
 
 
-PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
-
-
 class ItemTest(unittest.TestCase):
 
     def assertSortedEqual(self, first, second, msg=None):
@@ -280,14 +276,6 @@ class ItemMetaTest(unittest.TestCase):
         with mock.patch.object(base, '__new__', new_mock):
 
             class MyItem(Item):
-                if not PY36_PLUS:
-                    # This attribute is an internal attribute in Python 3.6+
-                    # and must be propagated properly. See
-                    # https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object
-                    # In <3.6, we add a dummy attribute just to ensure the
-                    # __new__ method propagates it correctly.
-                    __classcell__ = object()
-
                 def f(self):
                     # For rationale of this see:
                     # https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222

tests/test_proxy_connect.py

@@ -7,7 +7,6 @@ from subprocess import Popen, PIPE
 from urllib.parse import urlsplit, urlunsplit
 from unittest import skipIf
 
-import pytest
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
@@ -58,8 +57,6 @@ def _wrong_credentials(proxy_url):
     return urlunsplit(bad_auth_proxy)
 
 
-@skipIf(sys.version_info < (3, 5, 4),
-        "requires mitmproxy < 3.0.0, which these tests do not support")
 @skipIf("pypy" in sys.executable,
         "mitmproxy does not support PyPy")
 @skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),
@@ -88,14 +85,6 @@ class ProxyConnectTestCase(TestCase):
             yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
         self._assert_got_response_code(200, log)
 
-    @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info >= (3, 6))
-    @defer.inlineCallbacks
-    def test_https_connect_tunnel_error(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl("https://localhost:99999/status?n=200")
-        self._assert_got_tunnel_error(log)
-
     @defer.inlineCallbacks
     def test_https_tunnel_auth_error(self):
         os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy'])

tox.ini

@@ -89,7 +89,7 @@ deps =
 basepython = python3
 deps =
     {[pinned]deps}
-    # First lxml version that includes a Windows wheel for Python 3.5, so we do
+    # First lxml version that includes a Windows wheel for Python 3.6, so we do
     # not need to build lxml from sources in a CI Windows job:
     lxml==3.8.0