From ce8e515fa8960d0229132ac33fc5f8a6e43bbd6f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 3 Nov 2019 00:36:25 -0300 Subject: [PATCH] Remove six type wrappers --- scrapy/crawler.py | 2 +- scrapy/exporters.py | 4 ++-- scrapy/http/headers.py | 6 +++--- scrapy/http/request/__init__.py | 2 +- scrapy/http/response/text.py | 6 +++--- scrapy/linkextractors/htmlparser.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/linkextractors/sgml.py | 2 +- scrapy/settings/__init__.py | 10 +++++----- scrapy/spiders/crawl.py | 2 +- scrapy/spiders/sitemap.py | 4 ++-- scrapy/utils/iterators.py | 6 +++--- scrapy/utils/misc.py | 6 +++--- scrapy/utils/python.py | 14 +++++++------- 14 files changed, 34 insertions(+), 34 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 19b998e0d..b88df8f64 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -205,7 +205,7 @@ class CrawlerRunner(object): return self._create_crawler(crawler_or_spidercls) def _create_crawler(self, spidercls): - if isinstance(spidercls, six.string_types): + if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 2ff089ce9..e45317491 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -182,7 +182,7 @@ class XmlItemExporter(BaseItemExporter): for value in serialized_value: self._export_xml_field('value', value, depth=depth+1) self._beautify_indent(depth=depth) - elif isinstance(serialized_value, six.text_type): + elif isinstance(serialized_value, str): self.xg.characters(serialized_value) else: self.xg.characters(str(serialized_value)) @@ -321,7 +321,7 @@ class PythonItemExporter(BaseItemExporter): if is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode - if isinstance(value, (six.text_type, bytes)): + if isinstance(value, (str, bytes)): return encode_func(value, encoding=self.encoding) return value diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 62507eb19..5bbe7d72a 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -19,7 +19,7 @@ class Headers(CaselessDict): """Normalize values to bytes""" if value is None: value = [] - elif isinstance(value, (six.text_type, bytes)): + elif isinstance(value, (str, bytes)): value = [value] elif not hasattr(value, '__iter__'): value = [value] @@ -29,10 +29,10 @@ class Headers(CaselessDict): def _tobytes(self, x): if isinstance(x, bytes): return x - elif isinstance(x, six.text_type): + elif isinstance(x, str): return x.encode(self.encoding) elif isinstance(x, int): - return six.text_type(x).encode(self.encoding) + return str(x).encode(self.encoding) else: raise TypeError('Unsupported value type: {}'.format(type(x))) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index d09eaf849..ff7a44545 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -60,7 +60,7 @@ class Request(object_ref): return self._url def _set_url(self, url): - if not isinstance(url, six.string_types): + if not isinstance(url, str): raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) s = safe_url_string(url, self.encoding) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 69bcba577..65400803f 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -31,14 +31,14 @@ class TextResponse(Response): super(TextResponse, self).__init__(*args, **kwargs) def _set_url(self, url): - if isinstance(url, six.text_type): + if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: super(TextResponse, self)._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection - if isinstance(body, six.text_type): + if isinstance(body, str): if self._encoding is None: raise TypeError('Cannot convert unicode body - %s has no encoding' % type(self).__name__) @@ -158,7 +158,7 @@ class TextResponse(Response): def _url_from_selector(sel): # type: (parsel.Selector) -> str - if isinstance(sel.root, six.string_types): + if isinstance(sel.root, str): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 623732cee..2fec35799 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -42,7 +42,7 @@ class HtmlParserLinkExtractor(HTMLParser): ret = [] base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 5b7e709de..496ad3053 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()") def _nons(tag): - if isinstance(tag, six.string_types): + if isinstance(tag, str): if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: return tag.split('}')[-1] return tag diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 98bed15e9..a9dffcad3 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -49,7 +49,7 @@ class BaseSgmlLinkExtractor(SGMLParser): if base_url is None: base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in self.links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index d53b28895..c88c9c0e2 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -23,7 +23,7 @@ def get_settings_priority(priority): :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its numerical value, or directly returns a given numerical priority. """ - if isinstance(priority, six.string_types): + if isinstance(priority, str): return SETTINGS_PRIORITIES[priority] else: return priority @@ -173,7 +173,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or []) - if isinstance(value, six.string_types): + if isinstance(value, str): value = value.split(',') return list(value) @@ -194,7 +194,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or {}) - if isinstance(value, six.string_types): + if isinstance(value, str): value = json.loads(value) return dict(value) @@ -284,7 +284,7 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(module, six.string_types): + if isinstance(module, str): module = import_module(module) for key in dir(module): if key.isupper(): @@ -313,7 +313,7 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(values, six.string_types): + if isinstance(values, str): values = json.loads(values) if values is not None: if isinstance(values, BaseSettings): diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 03000ce54..59e2c5661 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -25,7 +25,7 @@ def _identity(request, response): def _get_method(method, spider): if callable(method): return method - elif isinstance(method, six.string_types): + elif isinstance(method, str): return getattr(spider, method, None) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 534c45c70..2917daf57 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -22,7 +22,7 @@ class SitemapSpider(Spider): super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: - if isinstance(c, six.string_types): + if isinstance(c, str): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow] @@ -86,7 +86,7 @@ class SitemapSpider(Spider): def regex(x): - if isinstance(x, six.string_types): + if isinstance(x, str): return re.compile(x) return x diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9693ba768..10481fe8d 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -60,7 +60,7 @@ class _StreamReader(object): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, six.text_type) + self._is_unicode = isinstance(self._text, str) def read(self, n=65535): self.read = self._read_unicode if self._is_unicode else self._read_string @@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - expected_types = (Response, six.text_type, six.binary_type) + expected_types = (Response, str, bytes) assert isinstance(obj, expected_types), \ "obj must be %s, not %s" % ( " or ".join(t.__name__ for t in expected_types), @@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True): return obj.text else: return obj.body.decode('utf-8') - elif isinstance(obj, six.text_type): + elif isinstance(obj, str): return obj if unicode else obj.encode('utf-8') else: return obj.decode('utf-8') if unicode else obj diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f638adb25..9a44f3576 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,7 @@ from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem -_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes +_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes def arg_to_iter(arg): @@ -83,7 +83,7 @@ def extract_regex(regex, text, encoding='utf-8'): * if the regex doesn't contain any group the entire regex matching is returned """ - if isinstance(regex, six.string_types): + if isinstance(regex, str): regex = re.compile(regex, re.UNICODE) try: @@ -92,7 +92,7 @@ def extract_regex(regex, text, encoding='utf-8'): strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) - if isinstance(text, six.text_type): + if isinstance(text, str): return [replace_entities(s, keep=['lt', 'amp']) for s in strings] else: return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 845c19fb9..18fee1964 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -68,7 +68,7 @@ def is_listlike(x): >>> is_listlike(range(5)) True """ - return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) + return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) def unique(list_, key=lambda x: x): @@ -87,9 +87,9 @@ def unique(list_, key=lambda x: x): def to_unicode(text, encoding=None, errors='strict'): """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" - if isinstance(text, six.text_type): + if isinstance(text, str): return text - if not isinstance(text, (bytes, six.text_type)): + if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -102,7 +102,7 @@ def to_bytes(text, encoding=None, errors='strict'): is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text - if not isinstance(text, six.string_types): + if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -138,7 +138,7 @@ def re_rsearch(pattern, text, chunk_size=1024): yield (text[offset:], offset) yield (text, 0) - if isinstance(pattern, six.string_types): + if isinstance(pattern, str): pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): @@ -301,9 +301,9 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): """ d = {} for k, v in dict(dct_or_tuples).items(): - k = k.encode(encoding) if isinstance(k, six.text_type) else k + k = k.encode(encoding) if isinstance(k, str) else k if not keys_only: - v = v.encode(encoding) if isinstance(v, six.text_type) else v + v = v.encode(encoding) if isinstance(v, str) else v d[k] = v return d