diff --git a/.bandit.yml b/.bandit.yml index 00554587a..243379b0b 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,13 +1,15 @@ skips: - B101 - B105 +- B301 - B303 - B306 - B307 - B311 - B320 - B321 -- B402 +- B402 # https://github.com/scrapy/scrapy/issues/4180 +- B403 - B404 - B406 - B410 diff --git a/conftest.py b/conftest.py index 24e31f130..64136b48d 100644 --- a/conftest.py +++ b/conftest.py @@ -1,12 +1,19 @@ +from pathlib import Path + import pytest +def _py_files(folder): + return (str(p) for p in Path(folder).rglob('*.py')) + + collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", + # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + *_py_files("tests/CrawlerProcess") ] - for line in open('tests/ignores.txt'): file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 8b2fef065..01986b594 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').get(), 'author': quote.xpath('span/small/text()').get(), + 'text': quote.css('span.text::text').get(), } next_page = response.css('li.next a::attr("href")').get() diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 3bef20bf3..da7a0022b 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function from time import time from collections import deque from twisted.web.server import Site, NOT_DONE_YET diff --git a/requirements-py3.txt b/requirements-py3.txt index 2c98e6f6d..28c649e28 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -2,7 +2,6 @@ parsel>=1.5.0 PyDispatcher>=2.0.5 Twisted>=17.9.0 w3lib>=1.17.0 -protego>=0.1.15 pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" @@ -14,5 +13,4 @@ cryptography>=2.0 # Earlier versions would fail to install cssselect>=0.9.1 lxml>=3.5.0 service_identity>=16.0.0 -six>=1.10.0 zope.interface>=4.1.3 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 230e5cee3..fb8357f3c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') del warnings # Apply monkey patches to fix issues in external libraries -from . import _monkeypatches +from scrapy import _monkeypatches del _monkeypatches from twisted import version as _txv diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 1f8067b35..f74f89bda 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,4 +1,4 @@ -from six.moves import copyreg +import copyreg # Undo what Twisted's perspective broker adds to pickle register diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index ce030cf75..3c2efe58f 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys import os import optparse diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 90c8d56a2..7bbe362e7 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,8 +1,7 @@ import sys import time import subprocess - -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode import scrapy from scrapy.commands import ScrapyCommand diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 724b4a1c4..8a22ebabe 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys from w3lib.url import is_url diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index d5498bb5c..adb01fa70 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,4 +1,3 @@ -from __future__ import print_function import os import shutil import string diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 422183ac1..54d7bb228 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,4 +1,3 @@ -from __future__ import print_function from scrapy.commands import ScrapyCommand diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ef8acd29c..ff6f1d8cd 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,4 +1,3 @@ -from __future__ import print_function import json import logging diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index ffe3aa2eb..603bafb9f 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,4 +1,3 @@ -from __future__ import print_function import json from scrapy.commands import ScrapyCommand diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 3b9f6eabb..e65131ae8 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import os import string diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 8651948f7..1516c5997 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import scrapy from scrapy.commands import ScrapyCommand from scrapy.utils.versions import scrapy_components_versions diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 24f6c2e77..e0d425874 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -4,7 +4,7 @@ from scrapy.item import BaseItem from scrapy.http import Request from scrapy.exceptions import ContractFail -from . import Contract +from scrapy.contracts import Contract # contracts diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 213268741..157dc3418 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -3,15 +3,14 @@ from time import time from datetime import datetime from collections import deque -import six from twisted.internet import reactor, defer, task from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache from scrapy import signals -from .middleware import DownloaderMiddlewareManager -from .handlers import DownloadHandlers +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.core.downloader.handlers import DownloadHandlers class Slot(object): @@ -188,7 +187,7 @@ class Downloader(object): def close(self): self._slot_gc_loop.stop() - for slot in six.itervalues(self.slots): + for slot in self.slots.values(): slot.close() def _slot_gc(self, age=60): diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 0b55d32fa..39a0b1f51 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,8 +1,9 @@ """Download handlers for different schemes""" import logging + from twisted.internet import defer -import six + from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -22,7 +23,7 @@ class DownloadHandlers(object): self._notconfigured = {} # remembers failed handlers handlers = without_none_values( crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) - for scheme, clspath in six.iteritems(handlers): + for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath self._load_handler(scheme, skip_lazy=True) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 39ed67a1a..aef231e82 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys: import re from io import BytesIO -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from twisted.internet import reactor from twisted.protocols.ftp import FTPClient, CommandFailed diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 6111e132a..52535bd8b 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,2 +1,4 @@ -from .http10 import HTTP10DownloadHandler -from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.core.downloader.handlers.http11 import ( + HTTP11DownloadHandler as HTTPDownloadHandler, +) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7d917cb74..63dedc19b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -2,10 +2,10 @@ import re import logging +import warnings from io import BytesIO from time import time -import warnings -from six.moves.urllib.parse import urldefrag +from urllib.parse import urldefrag from zope.interface import implementer from twisted.internet import defer, reactor, protocol diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 808d1bf21..e2a07bdef 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,9 @@ -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.boto import is_botocore -from .http import HTTPDownloadHandler +from scrapy.core.downloader.handlers.http import HTTPDownloadHandler def _get_boto_connection(): diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 7a6a4dfac..38608a429 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,6 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -import six - from twisted.internet import defer from scrapy.exceptions import _InvalidOutput @@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, response.__class__.__name__)) + (method.__self__.__class__.__name__, response.__class__.__name__)) if response: defer.returnValue(response) defer.returnValue((yield download_func(request=request, spider=spider))) @@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, response=response, spider=spider) if not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response))) + (method.__self__.__class__.__name__, type(response))) if isinstance(response, Request): defer.returnValue(response) defer.returnValue(response) @@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, exception=exception, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response))) + (method.__self__.__class__.__name__, type(response))) if response: defer.returnValue(response) defer.returnValue(_failure) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3fe13414a..798346f19 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,5 @@ from time import time -from six.moves.urllib.parse import urlparse, urlunparse, urldefrag +from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 00cee3ada..e2ade8256 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst """ from itertools import chain, islice -import six from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager @@ -37,8 +36,8 @@ class SpiderMiddlewareManager(MiddlewareManager): def scrape_response(self, scrape_func, response, request, spider): fname = lambda f: '%s.%s' % ( - six.get_method_self(f).__class__.__name__, - six.get_method_function(f).__name__) + f.__self__.__class__.__name__, + f.__func__.__name__) def process_spider_input(response): for method in self.methods['process_spider_input']: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 19b61dc7e..6c7eb737b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,7 +1,6 @@ -import pprint -import six -import signal import logging +import pprint +import signal import warnings from twisted.internet import reactor, defer @@ -22,6 +21,7 @@ from scrapy.utils.log import ( get_scrapy_root_handler, install_scrapy_root_handler) from scrapy import signals + logger = logging.getLogger(__name__) @@ -206,7 +206,7 @@ class CrawlerRunner(object): return self._create_crawler(crawler_or_spidercls) def _create_crawler(self, spidercls): - if isinstance(spidercls, six.string_types): + if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index ba50793bb..7a140fcad 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import re import logging -import six from w3lib import html from scrapy.exceptions import NotConfigured @@ -67,7 +65,7 @@ class AjaxCrawlMiddleware(object): # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile(six.u(r'')) +_ajax_crawlable_re = re.compile(r'') def _has_ajaxcrawlable_meta(text): diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index aeb7578b8..d8dabdf13 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,13 +1,12 @@ import logging from collections import defaultdict -import six - from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) @@ -82,8 +81,10 @@ class CookiesMiddleware(object): def _get_request_cookies(self, jar, request): if isinstance(request.cookies, dict): - cookie_list = [{'name': k, 'value': v} for k, v in \ - six.iteritems(request.cookies)] + cookie_list = [ + {'name': k, 'value': v} + for k, v in request.cookies.items() + ] else: cookie_list = request.cookies diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index e2d73f347..fcea38ef5 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive. import bz2 import gzip -from io import BytesIO -import zipfile -import tarfile import logging +import tarfile +import zipfile +from io import BytesIO from tempfile import mktemp -import six - from scrapy.responsetypes import responsetypes + logger = logging.getLogger(__name__) @@ -75,7 +74,7 @@ class DecompressionMiddleware(object): if not response.body: return response - for fmt, func in six.iteritems(self._formats): + for fmt, func in self._formats.items(): new_response = func(response) if new_response: logger.debug('Decompressed response with format: %(responsefmt)s', diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 5e4542b6c..814ce78fe 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,8 +1,6 @@ import base64 -from urllib.request import _parse_proxy - -from six.moves.urllib.parse import unquote, urlunparse -from six.moves.urllib.request import getproxies, proxy_bypass +from urllib.parse import unquote, urlunparse +from urllib.request import getproxies, proxy_bypass, _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index b73f864dd..77cb5aa94 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,5 +1,5 @@ import logging -from six.moves.urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string @@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured + logger = logging.getLogger(__name__) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 4d95eb847..ea6a4cfc3 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,4 +1,3 @@ -from __future__ import print_function import os import logging diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 3defafd60..fcb55da67 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -6,15 +6,14 @@ import csv import io import pprint import marshal -import six -from six.moves import cPickle as pickle +import warnings +import pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem from scrapy.exceptions import ScrapyDeprecationWarning -import warnings __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', @@ -60,9 +59,9 @@ class BaseItemExporter(object): include_empty = self.export_empty_fields if self.fields_to_export is None: if include_empty and not isinstance(item, dict): - field_iter = six.iterkeys(item.fields) + field_iter = item.fields.keys() else: - field_iter = six.iterkeys(item) + field_iter = item.keys() else: if include_empty: field_iter = self.fields_to_export @@ -180,7 +179,7 @@ class XmlItemExporter(BaseItemExporter): for value in serialized_value: self._export_xml_field('value', value, depth=depth+1) self._beautify_indent(depth=depth) - elif isinstance(serialized_value, six.text_type): + elif isinstance(serialized_value, str): self.xg.characters(serialized_value) else: self.xg.characters(str(serialized_value)) @@ -319,12 +318,12 @@ class PythonItemExporter(BaseItemExporter): if is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode - if isinstance(value, (six.text_type, bytes)): + if isinstance(value, (str, bytes)): return encode_func(value, encoding=self.encoding) return value def _serialize_dict(self, value): - for key, val in six.iteritems(value): + for key, val in value.items(): key = to_bytes(key) if self.binary else key yield key, self._serialize_value(val) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e2492d506..11a63ced2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,7 +10,7 @@ import logging import posixpath from tempfile import NamedTemporaryFile from datetime import datetime -from six.moves.urllib.parse import urlparse, unquote +from urllib.parse import urlparse, unquote from ftplib import FTP from zope.interface import Interface, implementer diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 11403957c..91850683f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,14 +1,12 @@ -from __future__ import print_function - import gzip import logging import os +import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module from time import time from weakref import WeakKeyDictionary -from six.moves import cPickle as pickle from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers, Response diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 263d8ce4c..892aa8a86 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst """ import gc -import six from scrapy import signals from scrapy.exceptions import NotConfigured @@ -28,7 +27,7 @@ class MemoryDebugger(object): def spider_closed(self, spider, reason): gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider) - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if not wdict: continue self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 8ba770ec0..2c8e46914 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,5 +1,5 @@ import os -from six.moves import cPickle as pickle +import pickle from scrapy import signals from scrapy.exceptions import NotConfigured diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 60a14c6f8..0903fd4f8 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,7 +1,6 @@ import time -from six.moves.http_cookiejar import ( - CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE -) +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE + from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index f3b46b994..dcaaeddfa 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,4 +1,3 @@ -import six from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode @@ -19,7 +18,7 @@ class Headers(CaselessDict): """Normalize values to bytes""" if value is None: value = [] - elif isinstance(value, (six.text_type, bytes)): + elif isinstance(value, (str, bytes)): value = [value] elif not hasattr(value, '__iter__'): value = [value] @@ -29,10 +28,10 @@ class Headers(CaselessDict): def _tobytes(self, x): if isinstance(x, bytes): return x - elif isinstance(x, six.text_type): + elif isinstance(x, str): return x.encode(self.encoding) elif isinstance(x, int): - return six.text_type(x).encode(self.encoding) + return str(x).encode(self.encoding) else: raise TypeError('Unsupported value type: {}'.format(type(x))) @@ -68,9 +67,6 @@ class Headers(CaselessDict): self[key] = lst def items(self): - return list(self.iteritems()) - - def iteritems(self): return ((k, self.getlist(k)) for k in self.keys()) def values(self): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 76a428199..b5c8e1a9a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,7 +4,6 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ -import six from w3lib.url import safe_url_string from scrapy.http.headers import Headers @@ -60,7 +59,7 @@ class Request(object_ref): return self._url def _set_url(self, url): - if not isinstance(url, six.string_types): + if not isinstance(url, str): raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) s = safe_url_string(url, self.encoding) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b6feede07..af02c8484 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin, urlencode +from urllib.parse import urljoin, urlencode import lxml.html from parsel.selector import create_root_node @@ -208,7 +207,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such xpath = u'.//*' + \ - u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata)) + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bd09f7534..811d3ad6b 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from six.moves import xmlrpc_client as xmlrpclib +import xmlrpc.client as xmlrpclib from scrapy.http.request import Request from scrapy.utils.python import get_func_args diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index a81404afb..64e9c6c20 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from scrapy.http.request import Request from scrapy.http.headers import Headers diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 37f450e54..1079fd6e8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ @@ -31,14 +30,14 @@ class TextResponse(Response): super(TextResponse, self).__init__(*args, **kwargs) def _set_url(self, url): - if isinstance(url, six.text_type): + if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: super(TextResponse, self)._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection - if isinstance(body, six.text_type): + if isinstance(body, str): if self._encoding is None: raise TypeError('Cannot convert unicode body - %s has no encoding' % type(self).__name__) @@ -158,7 +157,7 @@ class TextResponse(Response): def _url_from_selector(sel): # type: (parsel.Selector) -> str - if isinstance(sel.root, six.string_types): + if isinstance(sel.root, str): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): diff --git a/scrapy/item.py b/scrapy/item.py index 32f9b2ebb..1d39b48b2 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -10,8 +10,6 @@ from copy import deepcopy from pprint import pformat from warnings import warn -import six - from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref @@ -78,7 +76,7 @@ class DictItem(MutableMapping, BaseItem): def __init__(self, *args, **kwargs): self._values = {} if args or kwargs: # avoid creating dict for most common case - for k, v in six.iteritems(dict(*args, **kwargs)): + for k, v in dict(*args, **kwargs).items(): self[k] = v def __getitem__(self, key): @@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem): return deepcopy(self) -@six.add_metaclass(ItemMeta) -class Item(DictItem): +class Item(DictItem, metaclass=ItemMeta): pass diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 8c3693f04..4a3e74fbe 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,8 +6,8 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse from parsel.csstranslator import HTMLTranslator from w3lib.url import canonicalize_url @@ -118,4 +118,4 @@ class FilteringLinkExtractor(object): # Top-level imports -from .lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 27978a8a1..0425d4340 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -2,9 +2,8 @@ HTMLParser-based link extractor """ import warnings -import six -from six.moves.html_parser import HTMLParser -from six.moves.urllib.parse import urljoin +from html.parser import HTMLParser +from urllib.parse import urljoin from w3lib.url import safe_url_string from w3lib.html import strip_html5_whitespace @@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser): ret = [] base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 890c019c8..cb55e805a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,8 +1,7 @@ """ Link extractor based on lxml.html """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import lxml.etree as etree from w3lib.html import strip_html5_whitespace @@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()") def _nons(tag): - if isinstance(tag, six.string_types): + if isinstance(tag, str): if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: return tag.split('}')[-1] return tag diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index e689b4727..3f2557248 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -1,10 +1,11 @@ import re -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link -from .sgml import SgmlLinkExtractor +from scrapy.linkextractors.sgml import SgmlLinkExtractor + linkre = re.compile( "|\s.*?>)(.*?)<[/ ]?a>", diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 8940a4d77..2ba6bca45 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -1,9 +1,8 @@ """ SGMLParser-based Link extractors """ -import six -from six.moves.urllib.parse import urljoin import warnings +from urllib.parse import urljoin from sgmllib import SGMLParser from w3lib.url import safe_url_string, canonicalize_url @@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser): if base_url is None: base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in self.links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/mail.py b/scrapy/mail.py index d24de2212..891bb5e09 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,21 +3,21 @@ Mail sending helpers See documentation in docs/topics/email.rst """ -from io import BytesIO import logging - -from email.utils import COMMASPACE, formatdate -from six.moves.email_mime_multipart import MIMEMultipart -from six.moves.email_mime_text import MIMEText -from six.moves.email_mime_base import MIMEBase -from email.mime.nonmultipart import MIMENonMultipart from email import encoders as Encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.text import MIMEText +from email.utils import COMMASPACE, formatdate +from io import BytesIO from twisted.internet import defer, reactor, ssl from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes + logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8d74c5011..6d55c8980 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -5,16 +5,14 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib -from io import BytesIO +import logging import mimetypes import os -import os.path import time -import logging -from email.utils import parsedate_tz, mktime_tz -from six.moves.urllib.parse import urlparse from collections import defaultdict -import six +from email.utils import parsedate_tz, mktime_tz +from io import BytesIO +from urllib.parse import urlparse from twisted.internet import defer, threads @@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str from scrapy.utils.boto import is_botocore from scrapy.utils.datatypes import CaselessDict + logger = logging.getLogger(__name__) @@ -153,14 +152,14 @@ class S3FilesStore(object): Bucket=self.bucket, Key=key_name, Body=buf, - Metadata={k: str(v) for k, v in six.iteritems(meta or {})}, + Metadata={k: str(v) for k, v in (meta or {}).items()}, ACL=self.POLICY, **extra) else: b = self._get_boto_bucket() k = b.new_key(key_name) if meta: - for metakey, metavalue in six.iteritems(meta): + for metakey, metavalue in meta.items(): k.set_metadata(metakey, str(metavalue)) h = self.HEADERS.copy() if headers: @@ -201,7 +200,7 @@ class S3FilesStore(object): 'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation', }) extra = {} - for key, value in six.iteritems(headers): + for key, value in headers.items(): try: kwarg = mapping[key] except KeyError: @@ -249,7 +248,7 @@ class GCSFilesStore(object): def persist_file(self, path, buf, info, meta=None, headers=None): blob = self.bucket.blob(self.prefix + path) blob.cache_control = self.CACHE_CONTROL - blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})} + blob.metadata = {k: str(v) for k, v in (meta or {}).items()} return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e77cef4ff..e9c6b759c 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst import functools import hashlib from io import BytesIO -import six from PIL import Image @@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline): image, buf = self.convert_image(orig_image) yield path, image, buf - for thumb_id, size in six.iteritems(self.thumbs): + for thumb_id, size in self.thumbs.items(): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 95dca9a3f..c174addf9 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import functools import logging from collections import defaultdict diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index de62276c8..91d309147 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,11 +2,9 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ -from __future__ import absolute_import from mimetypes import MimeTypes from pkgutil import get_data from io import StringIO -import six from scrapy.http import Response from scrapy.utils.misc import load_object @@ -37,7 +35,7 @@ class ResponseTypes(object): self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types').decode('utf8') self.mimetypes.readfp(StringIO(mimedata)) - for mimetype, cls in six.iteritems(self.CLASSES): + for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index f0f9c59dc..0a9af3a62 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,7 +1,6 @@ import sys import logging from abc import ABCMeta, abstractmethod -from six import with_metaclass from scrapy.utils.python import to_unicode @@ -26,7 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): return robotstxt_body -class RobotParser(with_metaclass(ABCMeta)): +class RobotParser(metaclass=ABCMeta): @classmethod @abstractmethod def from_crawler(cls, crawler, robotstxt_body): @@ -56,7 +55,7 @@ class RobotParser(with_metaclass(ABCMeta)): class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): - from six.moves.urllib_robotparser import RobotFileParser + from urllib.robotparser import RobotFileParser self.spider = spider robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp = RobotFileParser() diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c871e86e0..b6133619c 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,4 +1,3 @@ -import six import json import copy from collections.abc import MutableMapping @@ -23,7 +22,7 @@ def get_settings_priority(priority): :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its numerical value, or directly returns a given numerical priority. """ - if isinstance(priority, six.string_types): + if isinstance(priority, str): return SETTINGS_PRIORITIES[priority] else: return priority @@ -173,7 +172,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or []) - if isinstance(value, six.string_types): + if isinstance(value, str): value = value.split(',') return list(value) @@ -194,7 +193,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or {}) - if isinstance(value, six.string_types): + if isinstance(value, str): value = json.loads(value) return dict(value) @@ -284,7 +283,7 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(module, six.string_types): + if isinstance(module, str): module = import_module(module) for key in dir(module): if key.isupper(): @@ -313,14 +312,14 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(values, six.string_types): + if isinstance(values, str): values = json.loads(values) if values is not None: if isinstance(values, BaseSettings): - for name, value in six.iteritems(values): + for name, value in values.items(): self.set(name, value, values.getpriority(name)) else: - for name, value in six.iteritems(values): + for name, value in values.items(): self.set(name, value, priority) def delete(self, name, priority='project'): @@ -377,7 +376,7 @@ class BaseSettings(MutableMapping): def _to_dict(self): return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) - for k, v in six.iteritems(self)} + for k, v in self.items()} def copy_to_dict(self): """ @@ -445,7 +444,7 @@ class Settings(BaseSettings): self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities - for name, val in six.iteritems(self): + for name, val in self.items(): if isinstance(val, dict): self.set(name, BaseSettings(val, 'default'), 'default') self.update(values, priority) diff --git a/scrapy/shell.py b/scrapy/shell.py index 80b625633..a649d555f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,8 +3,6 @@ See documentation in docs/topics/shell.rst """ -from __future__ import print_function - import os import signal import warnings diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 9a160f62e..481d97e9a 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import from pydispatch import dispatcher from scrapy.utils import signal as _signal diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 7478faa78..3beca4060 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from collections import defaultdict import traceback import warnings diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c76e4d5a2..dce2b3598 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,8 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlparse import warnings +from urllib.parse import urlparse from w3lib.url import safe_url_string diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 8d15dfceb..9429f6cb2 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -57,6 +57,11 @@ class Spider(object_ref): def start_requests(self): cls = self.__class__ + if not self.start_urls and hasattr(self, 'start_url'): + raise AttributeError( + "Crawling could not start: 'start_urls' not found " + "or empty (but found 'start_url' attribute instead, " + "did you miss an 's'?)") if method_is_overridden(cls, Spider, 'make_requests_from_url'): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 03000ce54..a5eb1a518 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst import copy import warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor @@ -25,7 +23,7 @@ def _identity(request, response): def _get_method(method, spider): if callable(method): return method - elif isinstance(method, six.string_types): + elif isinstance(method, str): return getattr(spider, method, None) diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 534c45c70..d368c7108 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,6 +1,5 @@ import re import logging -import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse @@ -22,7 +21,7 @@ class SitemapSpider(Spider): super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: - if isinstance(c, six.string_types): + if isinstance(c, str): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow] @@ -86,7 +85,7 @@ class SitemapSpider(Spider): def regex(x): - if isinstance(x, six.string_types): + if isinstance(x, str): return re.compile(x) return x diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 30cc926e5..d5d3be67e 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -from six.moves import cPickle as pickle +import pickle from queuelib import queue diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 5bbda6e27..cdbe21942 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,5 +1,6 @@ import random -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode + from twisted.web.server import Site from twisted.web.resource import Resource from twisted.internet import reactor diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index b76d5e56e..12321caa5 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,7 +1,5 @@ """Boto/botocore helpers""" -from __future__ import absolute_import - from scrapy.exceptions import NotConfigured diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 561bb72fc..7a15e77ff 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,11 +1,9 @@ -from configparser import ConfigParser import os import sys import numbers +from configparser import ConfigParser from operator import itemgetter -import six - from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values @@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _map_keys(compdict): if isinstance(compdict, BaseSettings): compbs = BaseSettings() - for k, v in six.iteritems(compdict): + for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: raise ValueError('Some paths in {!r} convert to the same ' @@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath): return compbs else: _check_components(compdict) - return {convert(k): v for k, v in six.iteritems(compdict)} + return {convert(k): v for k, v in compdict.items()} def _validate_values(compdict): """Fail if a value in the components dict is not a real number or None.""" - for name, value in six.iteritems(compdict): + for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): raise ValueError('Invalid value {} for component {}, please provide ' \ 'a real number or None instead'.format(value, name)) @@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) - return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] + return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] def arglist_to_dict(arglist): diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index 7fb25a71d..16639356e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,10 +1,9 @@ import argparse import warnings from shlex import split +from http.cookies import SimpleCookie +from urllib.parse import urlparse -from six.moves.http_cookies import SimpleCookie -from six.moves.urllib.parse import urlparse -from six import iteritems from w3lib.http import basic_auth_header @@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): name = name.strip() val = val.strip() if name.title() == 'Cookie': - for name, morsel in iteritems(SimpleCookie(val)): + for name, morsel in SimpleCookie(val).items(): cookies[name] = morsel.value else: headers.append((name, val)) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 39d389fa6..ffd1537c3 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -5,12 +5,10 @@ Python Standard Library. This module must not depend on any module outside the Standard Library. """ -import copy import collections -from collections.abc import Mapping +import copy import warnings - -import six +from collections.abc import Mapping from scrapy.exceptions import ScrapyDeprecationWarning @@ -151,7 +149,7 @@ class MultiValueDict(dict): self.setlistdefault(key, []).append(value) except TypeError: raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary") - for key, value in six.iteritems(kwargs): + for key, value in kwargs.items(): self.setlistdefault(key, []).append(value) @@ -238,65 +236,6 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class MergeDict(object): - """ - A simple class for creating new "virtual" dictionaries that actually look - up values in more than one dictionary, passed in the ``__init__`` method. - - If a key appears in more than one of the given dictionaries, only the - first occurrence will be used. - """ - def __init__(self, *dicts): - warnings.warn( - "scrapy.utils.datatypes.MergeDict is deprecated in favor " - "of collections.ChainMap (introduced in Python 3.3)", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.dicts = dicts - - def __getitem__(self, key): - for dict_ in self.dicts: - try: - return dict_[key] - except KeyError: - pass - raise KeyError - - def __copy__(self): - return self.__class__(*self.dicts) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def getlist(self, key): - for dict_ in self.dicts: - if key in dict_.keys(): - return dict_.getlist(key) - return [] - - def items(self): - item_list = [] - for dict_ in self.dicts: - item_list.extend(dict_.items()) - return item_list - - def has_key(self, key): - for dict_ in self.dicts: - if key in dict_: - return True - return False - - __contains__ = has_key - - def copy(self): - """Returns a copy of this object.""" - return self.__copy__() - - class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index 91ebdae11..9735220ef 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -2,7 +2,6 @@ pprint and pformat wrappers with colorization support """ -from __future__ import print_function import sys from pprint import pformat as pformat_ diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index b2be0a901..c8d4391b1 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,8 +1,7 @@ """Helper functions for scrapy.http objects (Request, Response)""" import weakref - -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse _urlparse_cache = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9693ba768..3c0cb68c3 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,13 +1,13 @@ -import re import csv -from io import StringIO import logging -import six +import re +from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode + logger = logging.getLogger(__name__) @@ -60,7 +60,7 @@ class _StreamReader(object): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, six.text_type) + self._is_unicode = isinstance(self._text, str) def read(self, n=65535): self.read = self._read_unicode if self._is_unicode else self._read_string @@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - expected_types = (Response, six.text_type, six.binary_type) + expected_types = (Response, str, bytes) assert isinstance(obj, expected_types), \ "obj must be %s, not %s" % ( " or ".join(t.__name__ for t in expected_types), @@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True): return obj.text else: return obj.body.decode('utf-8') - elif isinstance(obj, six.text_type): + elif isinstance(obj, str): return obj if unicode else obj.encode('utf-8') else: return obj.decode('utf-8') if unicode else obj diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b74f34451..9955fb1e7 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -6,14 +6,13 @@ from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules -import six from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem -_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes +_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes def arg_to_iter(arg): @@ -83,7 +82,7 @@ def extract_regex(regex, text, encoding='utf-8'): * if the regex doesn't contain any group the entire regex matching is returned """ - if isinstance(regex, six.string_types): + if isinstance(regex, str): regex = re.compile(regex, re.UNICODE) try: @@ -92,7 +91,7 @@ def extract_regex(regex, text, encoding='utf-8'): strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) - if isinstance(text, six.text_type): + if isinstance(text, str): return [replace_entities(s, keep=['lt', 'amp']) for s in strings] else: return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index f87d5a803..7a7aec9be 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,5 +1,3 @@ - -from __future__ import absolute_import import signal from twisted.internet import reactor diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 1cbda141a..f28c2eaa1 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,5 @@ import os -from six.moves import cPickle as pickle +import pickle import warnings from importlib import import_module @@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning + ENVVAR = 'SCRAPY_SETTINGS_MODULE' DATADIR_CFG_SECTION = 'datadir' diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 138e86d37..8d829c5a5 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,7 +7,6 @@ import re import inspect import weakref import errno -import six from functools import partial, wraps from itertools import chain import sys @@ -65,10 +64,10 @@ def is_listlike(x): True >>> is_listlike((x for x in range(3))) True - >>> is_listlike(six.moves.xrange(5)) + >>> is_listlike(range(5)) True """ - return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) + return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) def unique(list_, key=lambda x: x): @@ -87,9 +86,9 @@ def unique(list_, key=lambda x: x): def to_unicode(text, encoding=None, errors='strict'): """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" - if isinstance(text, six.text_type): + if isinstance(text, str): return text - if not isinstance(text, (bytes, six.text_type)): + if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -102,7 +101,7 @@ def to_bytes(text, encoding=None, errors='strict'): is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text - if not isinstance(text, six.string_types): + if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -138,7 +137,7 @@ def re_rsearch(pattern, text, chunk_size=1024): yield (text[offset:], offset) yield (text, 0) - if isinstance(pattern, six.string_types): + if isinstance(pattern, str): pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): @@ -162,7 +161,7 @@ def memoizemethod_noargs(method): return new_method -_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} +_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} @@ -301,10 +300,10 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): dict or a list of tuples, like any dict ``__init__`` method supports. """ d = {} - for k, v in six.iteritems(dict(dct_or_tuples)): - k = k.encode(encoding) if isinstance(k, six.text_type) else k + for k, v in dict(dct_or_tuples).items(): + k = k.encode(encoding) if isinstance(k, str) else k if not keys_only: - v = v.encode(encoding) if isinstance(v, six.text_type) else v + v = v.encode(encoding) if isinstance(v, str) else v d[k] = v return d @@ -346,7 +345,7 @@ def without_none_values(iterable): value ``None`` have been removed. """ try: - return {k: v for k, v in six.iteritems(iterable) if v is not None} + return {k: v for k, v in iterable.items() if v is not None} except AttributeError: return type(iterable)((v for v in iterable if v is not None)) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 495564ac0..749bbc387 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,8 +1,6 @@ """ Helper functions for serializing (and deserializing) requests. """ -import six - from scrapy.http import Request from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object @@ -87,12 +85,12 @@ def _mangle_private_name(obj, func, name): def _find_method(obj, func): if obj: try: - func_self = six.get_method_self(func) + func_self = func.__self__ except AttributeError: # func has no __self__ pass else: if func_self is obj: - name = six.get_method_function(func).__name__ + name = func.__func__.__name__ if _is_private_method(name): return _mangle_private_name(obj, func, name) return name diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 0fce5a2e1..356753ab5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -3,16 +3,15 @@ This module provides some useful functions for working with scrapy.http.Request objects """ -from __future__ import print_function import hashlib import weakref -from six.moves.urllib.parse import urlunparse +from urllib.parse import urlunparse from w3lib.http import basic_auth_header -from scrapy.utils.python import to_bytes, to_unicode - from w3lib.url import canonicalize_url + from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes, to_unicode _fingerprint_cache = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 4742b3e13..2f10cf4de 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ +from urllib.parse import urljoin + import lxml.etree -from six.moves.urllib.parse import urljoin class Sitemap(object): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index bf4973fbf..4061d1ea3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,11 +1,10 @@ import logging import inspect -import six - from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter + logger = logging.getLogger(__name__) @@ -21,7 +20,7 @@ def iter_spider_classes(module): # singleton in scrapy.spider.spiders from scrapy.spiders import Spider - for obj in six.itervalues(vars(module)): + for obj in vars(module).values(): if inspect.isclass(obj) and \ issubclass(obj, Spider) and \ obj.__module__ == module.__name__ and \ diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 9754366df..307c25352 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,7 +2,6 @@ This module contains some assorted functions used in tests """ -from __future__ import absolute_import import os from importlib import import_module diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index f268e91ff..0f15cf60a 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import sys import os diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index e50a989b3..6f5c21624 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,5 +1,4 @@ -from __future__ import print_function -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from twisted.internet import reactor from twisted.web import server, resource, static, util diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index eed14c5a1..4842b95df 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,12 +9,10 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -from __future__ import print_function import weakref from time import time from operator import itemgetter from collections import defaultdict -import six NoneType = type(None) @@ -37,13 +35,13 @@ def format_live_refs(ignore=NoneType): """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() - for cls, wdict in sorted(six.iteritems(live_refs), + for cls, wdict in sorted(live_refs.items(), key=lambda x: x[0].__name__): if not wdict: continue if issubclass(cls, ignore): continue - oldest = min(six.itervalues(wdict)) + oldest = min(wdict.values()) s += "%-30s %6d oldest: %ds ago\n" % ( cls.__name__, len(wdict), now - oldest ) @@ -57,15 +55,15 @@ def print_live_refs(*a, **kw): def get_oldest(class_name): """Get the oldest object for a specific class name""" - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if cls.__name__ == class_name: if not wdict: break - return min(six.iteritems(wdict), key=itemgetter(1))[0] + return min(wdict.items(), key=itemgetter(1))[0] def iter_all(class_name): """Iterate over all objects of the same class by its class name""" - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if cls.__name__ == class_name: - return six.iterkeys(wdict) + return wdict.keys() diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 2c7b324a1..c9abb12d5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) +from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code diff --git a/setup.py b/setup.py index 8f5f14f0d..85d797f88 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ setup( 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', - 'six>=1.10.0', 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py new file mode 100644 index 000000000..5f6f1ae30 --- /dev/null +++ b/tests/CrawlerProcess/simple.py @@ -0,0 +1,15 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/mockserver.py b/tests/mockserver.py index d09fbc171..ae249e55a 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -3,9 +3,9 @@ import os import random import sys from subprocess import Popen, PIPE +from urllib.parse import urlencode from OpenSSL import SSL -from six.moves.urllib.parse import urlencode from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2ac434f41..d97c4b8ee 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,7 +1,7 @@ # Tests requirements jmespath mitmproxy; python_version >= '3.6' -mitmproxy==3.0.4; python_version < '3.6' +mitmproxy<4.0.0; python_version < '3.6' pytest pytest-cov pytest-twisted >= 1.11 diff --git a/tests/spiders.py b/tests/spiders.py index 2487ecc22..981bd2eb8 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking """ import time -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from scrapy.spiders import Spider from scrapy.http import Request diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 56cfe642a..909ea90e0 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,13 +1,12 @@ -from io import StringIO import json import os import pstats import shutil -import six -from subprocess import Popen, PIPE import sys import tempfile import unittest +from io import StringIO +from subprocess import Popen, PIPE from scrapy.utils.test import get_testenv @@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase): for char in ("'", "<", ">", 'u"'): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) + self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b2e358700..582e3d052 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,6 +1,5 @@ from unittest import TextTestResult -from six import get_unbound_function from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase): with MockServer() as mockserver: contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) - get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc - get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc + TestSameUrlSpider.parse_first.__doc__ = contract_doc + TestSameUrlSpider.parse_second.__doc__ = contract_doc crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) yield crawler.crawl() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3fc13eeb7..3307899b7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -140,7 +140,7 @@ class CrawlTestCase(TestCase): def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" - from six.moves.urllib.parse import urlencode + from urllib.parse import urlencode query = urlencode({'raw': '''\ HTTP/1.1 200 OK Server: Apache-Coyote/1.1 diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 3ac45ca1d..9410b0e7a 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,4 +1,7 @@ import logging +import os +import subprocess +import sys import warnings from pytest import raises, mark @@ -16,6 +19,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet +from scrapy.utils.test import get_testenv class BaseCrawlerTest(unittest.TestCase): @@ -281,3 +285,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase): with LogCapture() as log: yield runner.crawl(AsyncioSpider) log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio'))) + + +class CrawlerProcessSubprocess(unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + + def run_script(self, script_name): + script_path = os.path.join(self.script_dir, script_name) + args = (sys.executable, script_path) + p = subprocess.Popen(args, env=get_testenv(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + return stderr.decode('utf-8') + + def test_simple(self): + log = self.run_script('simple.py') + self.assertIn('Spider closed (finished)', log) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 8266bf35f..a1645ed96 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - from unittest import mock from twisted.internet import reactor, error diff --git a/tests/test_engine.py b/tests/test_engine.py index 30150391a..002c4e6bb 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -10,9 +10,10 @@ module with the ``runserver`` argument:: python test_engine.py runserver """ -from __future__ import print_function -import sys, os, re -from six.moves.urllib.parse import urlparse +import os +import re +import sys +from urllib.parse import urlparse from twisted.internet import reactor, defer from twisted.web import server, static, util diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 0046c5666..5d1f5c182 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,15 +1,13 @@ -from __future__ import absolute_import import re import json import marshal +import pickle import tempfile import unittest from io import BytesIO from datetime import datetime -from six.moves import cPickle as pickle import lxml.etree -import six from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -80,7 +78,7 @@ class BaseItemExporterTest(unittest.TestCase): ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') _, name = list(ie._get_serialized_fields(self.i))[0] - assert isinstance(name, six.text_type) + assert isinstance(name, str) self.assertEqual(name, u'John\xa3') def test_field_custom_serializer(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index ce3c4f059..2ca57c19d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,15 +1,15 @@ -from __future__ import absolute_import import os import csv import json import warnings -from io import BytesIO import tempfile import shutil import string +from io import BytesIO +from pathlib import Path from unittest import mock -from six.moves.urllib.parse import urljoin, urlparse, quote -from six.moves.urllib.request import pathname2url +from urllib.parse import urljoin, urlparse, quote +from urllib.request import pathname2url from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -28,8 +28,6 @@ from scrapy.extensions.feedexport import ( from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler from scrapy.utils.python import to_unicode -from pathlib import Path - class FileFeedStorageTest(unittest.TestCase): diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 0a9ed500a..45ddb42ba 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from unittest import TestCase from scrapy.http import Request, Response diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index c83cf3b66..cf3fc8496 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -86,9 +86,6 @@ class HeadersTest(unittest.TestCase): self.assertSortedEqual(h.items(), [(b'X-Forwarded-For', [b'ip1', b'ip2']), (b'Content-Type', [b'text/html'])]) - self.assertSortedEqual(h.iteritems(), - [(b'X-Forwarded-For', [b'ip1', b'ip2']), - (b'Content-Type', [b'text/html'])]) self.assertSortedEqual(h.values(), [b'ip2', b'text/html']) def test_update(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 9df6ff67b..a98aa1e6f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,12 +1,10 @@ import unittest import re import json -from unittest import mock -from urllib.parse import unquote_to_bytes +import xmlrpc.client import warnings - -from six.moves import xmlrpc_client as xmlrpclib -from six.moves.urllib.parse import urlparse, parse_qs +from unittest import mock +from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_unicode @@ -64,7 +62,7 @@ class RequestTest(unittest.TestCase): # headers must not be unicode h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' - for k, v in h.iteritems(): + for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: self.assertIsInstance(s, bytes) @@ -1220,7 +1218,7 @@ class XmlRpcRequestTest(RequestTest): r = self.request_class('http://scrapytest.org/rpc2', **kwargs) self.assertEqual(r.headers[b'Content-Type'], b'text/xml') self.assertEqual(r.body, - to_bytes(xmlrpclib.dumps(**kwargs), + to_bytes(xmlrpc.client.dumps(**kwargs), encoding=kwargs.get('encoding', 'utf-8'))) self.assertEqual(r.method, 'POST') self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8')) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 883c943da..79bb745cc 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unittest -import six from w3lib.encoding import resolve_encoding from scrapy.http import (Request, Response, TextResponse, HtmlResponse, @@ -102,7 +101,7 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r4.flags, []) def _assert_response_values(self, response, encoding, body): - if isinstance(body, six.text_type): + if isinstance(body, str): body_unicode = body body_bytes = body.encode(encoding) else: @@ -110,7 +109,7 @@ class BaseResponseTest(unittest.TestCase): body_bytes = body assert isinstance(response.body, bytes) - assert isinstance(response.text, six.text_type) + assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) self.assertEqual(response.body_as_unicode(), body_unicode) @@ -220,11 +219,11 @@ class TextResponseTest(BaseResponseTest): r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') # check body_as_unicode - self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type)) + self.assertTrue(isinstance(r1.body_as_unicode(), str)) self.assertEqual(r1.body_as_unicode(), unicode_string) # check response.text - self.assertTrue(isinstance(r1.text, six.text_type)) + self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) def test_encoding(self): diff --git a/tests/test_item.py b/tests/test_item.py index 49117ef04..30463a0f5 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -3,8 +3,6 @@ import unittest from unittest import mock from warnings import catch_warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta @@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): - class MyItem(six.with_metaclass(ItemMeta, Item)): + class MyItem(Item, metaclass=ItemMeta): def __init__(self, *args, **kwargs): # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: diff --git a/tests/test_loader.py b/tests/test_loader.py index 6bfc31dbf..579a85ff6 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,8 +1,6 @@ from functools import partial import unittest -import six - from scrapy.http import HtmlResponse from scrapy.item import Item, Field from scrapy.loader import ItemLoader @@ -157,7 +155,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper)) + self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) @@ -258,7 +256,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase) + name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() il.add_value('name', u'marta') @@ -266,7 +264,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase) + name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) il = ChildDefaultedItemLoader() il.add_value('name', u'marta') @@ -689,7 +687,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assertIsInstance(proc(['hello', 'world']), six.text_type) + self.assertIsInstance(proc(['hello', 'world']), str) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -704,12 +702,12 @@ class ProcessorsTest(unittest.TestCase): def test_mapcompose(self): def filter_world(x): return None if x == 'world' else x - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), [u'HELLO', u'THIS', u'IS', u'SCRAPY']) - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertRaises(ValueError, proc, [1]) proc = MapCompose(filter_world, lambda x: x + 1) self.assertRaises(ValueError, proc, 'hello') diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index d0b23a8c4..7d8c6ec7f 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -3,7 +3,6 @@ import unittest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase as TwistedTestCase -import six from scrapy.crawler import CrawlerRunner from scrapy.exceptions import DropItem @@ -60,7 +59,7 @@ class LogFormatterTestCase(unittest.TestCase): logkws = self.formatter.dropped(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() - assert all(isinstance(x, six.text_type) for x in lines) + assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, [u"Dropped: \u2018", '{}']) def test_error(self): @@ -80,7 +79,7 @@ class LogFormatterTestCase(unittest.TestCase): logkws = self.formatter.scraped(item, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() - assert all(isinstance(x, six.text_type) for x in lines) + assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bd40e4103..52f2b554e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,11 +1,11 @@ import os import random import time +from io import BytesIO from tempfile import mkdtemp from shutil import rmtree from unittest import mock -from six.moves.urllib.parse import urlparse -from six import BytesIO +from urllib.parse import urlparse from twisted.trial import unittest from twisted.internet import defer diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index ad958e25f..0d23f51cc 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 277455751..5f68cd448 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,20 +1,20 @@ import json import os import re -from subprocess import Popen, PIPE import sys +from subprocess import Popen, PIPE +from urllib.parse import urlsplit, urlunsplit import pytest -from six.moves.urllib.parse import urlsplit, urlunsplit from testfixtures import LogCapture - from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import get_crawler from scrapy.http import Request -from tests.spiders import SimpleSpider, SingleRequestSpider +from scrapy.utils.test import get_crawler + from tests.mockserver import MockServer +from tests.spiders import SimpleSpider, SingleRequestSpider class MitmProxy: diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 32e65bed5..fda44653a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,3 @@ -import six import unittest from unittest import mock @@ -10,7 +9,7 @@ from . import default_settings class SettingsGlobalFuncsTest(unittest.TestCase): def test_get_settings_priority(self): - for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES): + for prio_str, prio_num in SETTINGS_PRIORITIES.items(): self.assertEqual(get_settings_priority(prio_str), prio_num) self.assertEqual(get_settings_priority(99), 99) @@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {'three': 11, 'four': 21} attribute.set(new_dict, 10) self.assertIsInstance(attribute.value, BaseSettings) - six.assertCountEqual(self, attribute.value, new_dict) - six.assertCountEqual(self, original_settings, original_dict) + self.assertCountEqual(attribute.value, new_dict) + self.assertCountEqual(original_settings, original_dict) new_settings = BaseSettings({'five': 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - six.assertCountEqual(self, attribute.value, new_dict) + self.assertCountEqual(attribute.value, new_dict) attribute.set(new_settings, 10) - six.assertCountEqual(self, attribute.value, new_settings) + self.assertCountEqual(attribute.value, new_settings) def test_repr(self): self.assertEqual(repr(self.attribute), @@ -148,10 +147,10 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setmodule( 'tests.test_settings.default_settings', 10) - self.assertCountEqual(six.iterkeys(self.settings.attributes), - six.iterkeys(ctrl_attributes)) + self.assertCountEqual(self.settings.attributes.keys(), + ctrl_attributes.keys()) - for key in six.iterkeys(ctrl_attributes): + for key in ctrl_attributes.keys(): attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] self.assertEqual(attr.value, ctrl_attr.value) @@ -227,7 +226,7 @@ class BaseSettingsTest(unittest.TestCase): } settings = self.settings settings.attributes = {key: SettingsAttribute(value, 0) for key, value - in six.iteritems(test_configuration)} + in test_configuration.items()} self.assertTrue(settings.getbool('TEST_ENABLED1')) self.assertTrue(settings.getbool('TEST_ENABLED2')) @@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase): 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) s['TEST'].set(2, 200, 'cmdline') - six.assertCountEqual(self, s.getwithbase('TEST'), - {1: 1, 2: 200, 3: 30}) - six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE']) + self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30}) + self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE']) self.assertEqual(s.getwithbase('NONEXISTENT'), {}) def test_maxpriority(self): diff --git a/tests/test_spider.py b/tests/test_spider.py index aa43e3b3a..317a27076 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -384,6 +384,14 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) + def test_start_url(self): + spider = self.spider_class("example.com") + spider.start_url = 'https://www.example.com' + + with self.assertRaisesRegex(AttributeError, + r'^Crawling could not start.*$'): + list(spider.start_requests()) + class SitemapSpiderTest(SpiderTest): diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b97d9b675..992e60be2 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,13 +1,11 @@ from unittest import TestCase - -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse +import warnings from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.offsite import OffsiteMiddleware -from scrapy.spidermiddlewares.offsite import URLWarning +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning from scrapy.utils.test import get_crawler -import warnings class TestOffsiteMiddleware(TestCase): diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 2be6a1cd5..ecec6135d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from unittest import TestCase import warnings diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 91bbe43bc..fdc5df166 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,12 +1,12 @@ from unittest import TestCase -import six + import scrapy class ToplevelTestCase(TestCase): def test_version(self): - self.assertIs(type(scrapy.__version__), six.text_type) + self.assertIs(type(scrapy.__version__), str) def test_version_info(self): self.assertIs(type(scrapy.version_info), tuple) diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 22e39821c..bea0cf3e5 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse import unittest diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index c5655df7e..50e1bfd5f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,7 +1,6 @@ import unittest import warnings -from six import assertRaisesRegex from w3lib.http import basic_auth_header from scrapy import Request @@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) def test_too_few_arguments_error(self): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, r"too few arguments|the following arguments are required:\s*url", lambda: curl_to_request_kwargs("curl"), @@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, "Unrecognized options:.*--bar.*--baz", lambda: curl_to_request_kwargs( diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 53228fc6e..38a25778e 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase): self.assertIn(20, d) self.assertNotIn(15, d) - def test_six_range(self): - import six.moves - seq = six.moves.range(10**3, 10**6) - d = SequenceExclude(seq) - self.assertIn(10**2, d) - self.assertIn(10**7, d) - self.assertNotIn(10**4, d) - def test_range_step(self): seq = range(10, 20, 3) d = SequenceExclude(seq) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index d642ed3ed..49c2befb5 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -5,8 +5,6 @@ from twisted.python.failure import Failure from scrapy.utils.defer import mustbe_deferred, process_chain, \ process_chain_both, process_parallel, iter_errback -from six.moves import xrange - class MustbeDeferredTest(unittest.TestCase): def test_success_function(self): @@ -92,7 +90,7 @@ class IterErrbackTest(unittest.TestCase): def test_iter_errback_good(self): def itergood(): - for x in xrange(10): + for x in range(10): yield x errors = [] @@ -102,7 +100,7 @@ class IterErrbackTest(unittest.TestCase): def test_iter_errback_bad(self): def iterbad(): - for x in xrange(10): + for x in range(10): if x == 5: a = 1/0 yield x diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index ce04e7f29..159ef8f25 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import inspect import unittest from unittest import mock diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 2c3965bbc..cf8ad1f23 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,5 +1,5 @@ import unittest -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f16ef8110..9776dfb2a 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- import os -import six + from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata + FOOBAR_NL = u"foo\nbar" @@ -256,8 +257,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values()))) + self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) + self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) def test_csviter_delimiter(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 742e04803..2c23f3616 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import sys import logging import unittest diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 2d27d4b81..b79e0ac1c 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,10 +1,9 @@ -import gc import functools +import gc import operator +import platform import unittest from itertools import count -import platform -import six from warnings import catch_warnings from scrapy.utils.python import ( @@ -12,6 +11,7 @@ from scrapy.utils.python import ( WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) + __doctests__ = ['scrapy.utils.python'] @@ -205,12 +205,12 @@ class UtilsPythonTestCase(unittest.TestCase): if platform.python_implementation() == 'CPython': # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(six.text_type.split), []) + self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: self.assertEqual( - get_func_args(six.text_type.split, stripself=True), ['sep', 'maxsplit']) + get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ['obj']) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 3da95b95a..3e664fc74 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,4 +1,3 @@ -from __future__ import print_function import unittest from scrapy.http import Request from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \ diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index bea4dade3..6ebf290c0 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,12 +1,13 @@ import os import unittest -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import (response_httprepr, open_in_browser, get_meta_refresh, get_base_url, response_status_message) + __doctests__ = ['scrapy.utils.response'] diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 480a717e7..16e02f919 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,6 +1,7 @@ -import six import unittest +from io import StringIO from unittest import mock + from scrapy.utils import trackref @@ -38,12 +39,12 @@ Live References Bar 1 oldest: 0s ago ''') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_empty(self, stdout): trackref.print_live_refs() self.assertEqual(stdout.getvalue(), 'Live References\n\n\n') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): o1 = Foo() # NOQA trackref.print_live_refs() diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index c7bcaf88b..21e9a056a 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -5,6 +5,7 @@ from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, add_http_if_no_scheme, guess_scheme, strip_url) + __doctests__ = ['scrapy.utils.url'] diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 7b015ff8d..746367b41 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -3,7 +3,6 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ import os -import six import shutil import OpenSSL.SSL @@ -298,7 +297,7 @@ class WebClientTestCase(unittest.TestCase): def cleanup(passthrough): # Clean up the server which is hanging around not doing # anything. - connected = list(six.iterkeys(self.wrapper.protocols)) + connected = list(self.wrapper.protocols.keys()) # There might be nothing here if the server managed to already see # that the connection was lost. if connected: