diff --git a/requirements-py3.txt b/requirements-py3.txt index cd183a525..28c649e28 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -13,5 +13,4 @@ cryptography>=2.0 # Earlier versions would fail to install cssselect>=0.9.1 lxml>=3.5.0 service_identity>=16.0.0 -six>=1.10.0 zope.interface>=4.1.3 diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 78b802673..c618e9ffc 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -2,7 +2,6 @@ import re import logging -import six from w3lib import html from scrapy.exceptions import NotConfigured @@ -66,7 +65,7 @@ class AjaxCrawlMiddleware(object): # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile(six.u(r'')) +_ajax_crawlable_re = re.compile(r'') def _has_ajaxcrawlable_meta(text): """ >>> _has_ajaxcrawlable_meta('') diff --git a/scrapy/item.py b/scrapy/item.py index 4e0f0ac44..1d39b48b2 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -10,8 +10,6 @@ from copy import deepcopy from pprint import pformat from warnings import warn -import six - from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref @@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem): return deepcopy(self) -@six.add_metaclass(ItemMeta) -class Item(DictItem): +class Item(DictItem, metaclass=ItemMeta): pass diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 397924110..0a9af3a62 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,12 +1,13 @@ import sys import logging from abc import ABCMeta, abstractmethod -from six import with_metaclass from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) + def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: @@ -23,7 +24,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): robotstxt_body = '' return robotstxt_body -class RobotParser(with_metaclass(ABCMeta)): + +class RobotParser(metaclass=ABCMeta): @classmethod @abstractmethod def from_crawler(cls, crawler, robotstxt_body): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 18fee1964..d32ee5a3a 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,7 +7,6 @@ import re import inspect import weakref import errno -import six from functools import partial, wraps from itertools import chain import sys @@ -162,7 +161,7 @@ def memoizemethod_noargs(method): return new_method -_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} +_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} @deprecated("scrapy.utils.python.binary_is_text") diff --git a/setup.py b/setup.py index 8f5f14f0d..85d797f88 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ setup( 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', - 'six>=1.10.0', 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index c2b16bec6..7abb66b9c 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,7 @@ # Tests requirements jmespath mitmproxy; python_version >= '3.6' -mitmproxy==3.0.4; python_version < '3.6' +mitmproxy<4.0.0; python_version < '3.6' pytest pytest-cov pytest-twisted diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 56cfe642a..909ea90e0 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,13 +1,12 @@ -from io import StringIO import json import os import pstats import shutil -import six -from subprocess import Popen, PIPE import sys import tempfile import unittest +from io import StringIO +from subprocess import Popen, PIPE from scrapy.utils.test import get_testenv @@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase): for char in ("'", "<", ">", 'u"'): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) + self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b2e358700..582e3d052 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,6 +1,5 @@ from unittest import TextTestResult -from six import get_unbound_function from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase): with MockServer() as mockserver: contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) - get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc - get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc + TestSameUrlSpider.parse_first.__doc__ = contract_doc + TestSameUrlSpider.parse_second.__doc__ = contract_doc crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) yield crawler.crawl() diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 988c8a811..05cac617c 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -6,7 +6,7 @@ import json import xmlrpc.client as xmlrpclib import warnings from unittest import mock -from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse +from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_unicode diff --git a/tests/test_item.py b/tests/test_item.py index d98c63ddd..0da8fa1ac 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -3,8 +3,6 @@ import unittest from unittest import mock from warnings import catch_warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta @@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): - class MyItem(six.with_metaclass(ItemMeta, Item)): + class MyItem(Item, metaclass=ItemMeta): def __init__(self, *args, **kwargs): # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index dede4bf12..52f2b554e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,12 +1,12 @@ import os import random import time +from io import BytesIO from tempfile import mkdtemp from shutil import rmtree from unittest import mock from urllib.parse import urlparse -from six import BytesIO from twisted.trial import unittest from twisted.internet import defer diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index d5cbef6f5..fda44653a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,3 @@ -import six import unittest from unittest import mock @@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {'three': 11, 'four': 21} attribute.set(new_dict, 10) self.assertIsInstance(attribute.value, BaseSettings) - six.assertCountEqual(self, attribute.value, new_dict) - six.assertCountEqual(self, original_settings, original_dict) + self.assertCountEqual(attribute.value, new_dict) + self.assertCountEqual(original_settings, original_dict) new_settings = BaseSettings({'five': 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - six.assertCountEqual(self, attribute.value, new_dict) + self.assertCountEqual(attribute.value, new_dict) attribute.set(new_settings, 10) - six.assertCountEqual(self, attribute.value, new_settings) + self.assertCountEqual(attribute.value, new_settings) def test_repr(self): self.assertEqual(repr(self.attribute), @@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase): 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) s['TEST'].set(2, 200, 'cmdline') - six.assertCountEqual(self, s.getwithbase('TEST'), - {1: 1, 2: 200, 3: 30}) - six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE']) + self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30}) + self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE']) self.assertEqual(s.getwithbase('NONEXISTENT'), {}) def test_maxpriority(self): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index c5655df7e..50e1bfd5f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,7 +1,6 @@ import unittest import warnings -from six import assertRaisesRegex from w3lib.http import basic_auth_header from scrapy import Request @@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) def test_too_few_arguments_error(self): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, r"too few arguments|the following arguments are required:\s*url", lambda: curl_to_request_kwargs("curl"), @@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, "Unrecognized options:.*--bar.*--baz", lambda: curl_to_request_kwargs( diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 480a717e7..16e02f919 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,6 +1,7 @@ -import six import unittest +from io import StringIO from unittest import mock + from scrapy.utils import trackref @@ -38,12 +39,12 @@ Live References Bar 1 oldest: 0s ago ''') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_empty(self, stdout): trackref.print_live_refs() self.assertEqual(stdout.getvalue(), 'Live References\n\n\n') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): o1 = Foo() # NOQA trackref.print_live_refs()