RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'remove-six-code' of github.com:scrapy/scrapy into remove-six-code

This commit is contained in:
Eugenio Lacuesta 2019-11-08 22:17:12 -03:00
parent b6bbb28197 44f19df311
commit 07fa768d33
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
15 changed files with 28 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
requirements-py3.txt
View File

@ -13,5 +13,4 @@ cryptography>=2.0 # Earlier versions would fail to install
cssselect>=0.9.1
lxml>=3.5.0
service_identity>=16.0.0
six>=1.10.0
zope.interface>=4.1.3

3
scrapy/downloadermiddlewares/ajaxcrawl.py
View File

@ -2,7 +2,6 @@
import re
import logging
import six
from w3lib import html
from scrapy.exceptions import NotConfigured
@ -66,7 +65,7 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
def _has_ajaxcrawlable_meta(text):
"""
>>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment" content="!"/></head><body></body></html>')

5
scrapy/item.py
View File

@ -10,8 +10,6 @@ from copy import deepcopy
from pprint import pformat
from warnings import warn
import six
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
return deepcopy(self)
@six.add_metaclass(ItemMeta)
class Item(DictItem):
class Item(DictItem, metaclass=ItemMeta):
pass

6
scrapy/robotstxt.py
View File

@ -1,12 +1,13 @@
import sys
import logging
from abc import ABCMeta, abstractmethod
from six import with_metaclass
from scrapy.utils.python import to_unicode
logger = logging.getLogger(__name__)
def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
try:
if to_native_str_type:
@ -23,7 +24,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
robotstxt_body = ''
return robotstxt_body
class RobotParser(with_metaclass(ABCMeta)):
class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
def from_crawler(cls, crawler, robotstxt_body):

3
scrapy/utils/python.py
View File

@ -7,7 +7,6 @@ import re
import inspect
import weakref
import errno
import six
from functools import partial, wraps
from itertools import chain
import sys
@ -162,7 +161,7 @@ def memoizemethod_noargs(method):
return new_method
_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
@deprecated("scrapy.utils.python.binary_is_text")

1
setup.py
View File

@ -72,7 +72,6 @@ setup(
'pyOpenSSL>=16.2.0',
'queuelib>=1.4.2',
'service_identity>=16.0.0',
'six>=1.10.0',
'w3lib>=1.17.0',
'zope.interface>=4.1.3',
'protego>=0.1.15',

2
tests/requirements-py3.txt
View File

@ -1,7 +1,7 @@
# Tests requirements
jmespath
mitmproxy; python_version >= '3.6'
mitmproxy==3.0.4; python_version < '3.6'
mitmproxy<4.0.0; python_version < '3.6'
pytest
pytest-cov
pytest-twisted

7
tests/test_cmdline/__init__.py
View File

@ -1,13 +1,12 @@
from io import StringIO
import json
import os
import pstats
import shutil
import six
from subprocess import Popen, PIPE
import sys
import tempfile
import unittest
from io import StringIO
from subprocess import Popen, PIPE
from scrapy.utils.test import get_testenv
@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase):
for char in ("'", "<", ">", 'u"'):
settingsstr = settingsstr.replace(char, '"')
settingsdict = json.loads(settingsstr)
six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
self.assertEqual(200, settingsdict[EXT_PATH])

5
tests/test_contracts.py
View File

@ -1,6 +1,5 @@
from unittest import TextTestResult
from six import get_unbound_function
from twisted.internet import defer
from twisted.python import failure
from twisted.trial import unittest
@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase):
with MockServer() as mockserver:
contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
TestSameUrlSpider.parse_first.__doc__ = contract_doc
TestSameUrlSpider.parse_second.__doc__ = contract_doc
crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
yield crawler.crawl()

2
tests/test_http_request.py
View File

@ -6,7 +6,7 @@ import json
import xmlrpc.client as xmlrpclib
import warnings
from unittest import mock
from urllib.parse import parse_qs, unquote, unquote_to_bytes, urlparse
from urllib.parse import parse_qs, unquote_to_bytes, urlparse
from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
from scrapy.utils.python import to_bytes, to_unicode

4
tests/test_item.py
View File

@ -3,8 +3,6 @@ import unittest
from unittest import mock
from warnings import catch_warnings
import six
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase):
class ItemMetaClassCellRegression(unittest.TestCase):
def test_item_meta_classcell_regression(self):
class MyItem(six.with_metaclass(ItemMeta, Item)):
class MyItem(Item, metaclass=ItemMeta):
def __init__(self, *args, **kwargs):
# This call to super() trigger the __classcell__ propagation
# requirement. When not done properly raises an error:

2
tests/test_pipeline_files.py
View File

@ -1,12 +1,12 @@
import os
import random
import time
from io import BytesIO
from tempfile import mkdtemp
from shutil import rmtree
from unittest import mock
from urllib.parse import urlparse
from six import BytesIO
from twisted.trial import unittest
from twisted.internet import defer

14
tests/test_settings/__init__.py
View File

@ -1,4 +1,3 @@
import six
import unittest
from unittest import mock
@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase):
new_dict = {'three': 11, 'four': 21}
attribute.set(new_dict, 10)
self.assertIsInstance(attribute.value, BaseSettings)
six.assertCountEqual(self, attribute.value, new_dict)
six.assertCountEqual(self, original_settings, original_dict)
self.assertCountEqual(attribute.value, new_dict)
self.assertCountEqual(original_settings, original_dict)
new_settings = BaseSettings({'five': 12}, 0)
attribute.set(new_settings, 0) # Insufficient priority
six.assertCountEqual(self, attribute.value, new_dict)
self.assertCountEqual(attribute.value, new_dict)
attribute.set(new_settings, 10)
six.assertCountEqual(self, attribute.value, new_settings)
self.assertCountEqual(attribute.value, new_settings)
def test_repr(self):
self.assertEqual(repr(self.attribute),
@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase):
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
s['TEST'].set(2, 200, 'cmdline')
six.assertCountEqual(self, s.getwithbase('TEST'),
{1: 1, 2: 200, 3: 30})
six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
self.assertEqual(s.getwithbase('NONEXISTENT'), {})
def test_maxpriority(self):

7
tests/test_utils_curl.py
View File

@ -1,7 +1,6 @@
import unittest
import warnings
from six import assertRaisesRegex
from w3lib.http import basic_auth_header
from scrapy import Request
@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
def test_too_few_arguments_error(self):
assertRaisesRegex(
self,
self.assertRaisesRegex(
ValueError,
r"too few arguments|the following arguments are required:\s*url",
lambda: curl_to_request_kwargs("curl"),
@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
# case 2: ignore_unknown_options=False (raise exception):
assertRaisesRegex(
self,
self.assertRaisesRegex(
ValueError,
"Unrecognized options:.*--bar.*--baz",
lambda: curl_to_request_kwargs(

7
tests/test_utils_trackref.py
View File

@ -1,6 +1,7 @@
import six
import unittest
from io import StringIO
from unittest import mock
from scrapy.utils import trackref
@ -38,12 +39,12 @@ Live References
Bar 1 oldest: 0s ago
''')
@mock.patch('sys.stdout', new_callable=six.StringIO)
@mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_empty(self, stdout):
trackref.print_live_refs()
self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
@mock.patch('sys.stdout', new_callable=six.StringIO)
@mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_with_objects(self, stdout):
o1 = Foo() # NOQA
trackref.print_live_refs()