From 09e27d2d2e04fd44d9e49fbd328805ad9eca594f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 18:08:07 -0300 Subject: [PATCH 01/10] [Tests] Monkey patch unittest.TestCase to prevent DeprecationWarning(s) --- tests/__init__.py | 13 +++++++++++++ tests/test_exporters.py | 2 +- tests/test_http_headers.py | 16 ++++++++-------- tests/test_http_request.py | 6 +++--- tests/test_http_response.py | 18 +++++++++--------- tests/test_selector.py | 2 +- tests/test_spider.py | 2 +- tests/test_spiderloader/__init__.py | 4 ++-- 8 files changed, 38 insertions(+), 25 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index a54367f8c..e55c77734 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -35,3 +35,16 @@ def get_testdata(*paths): path = os.path.join(tests_datadir, *paths) with open(path, 'rb') as f: return f.read() + + +# FIXME: delete after dropping py2 support +# Monkey patch the unittest module to prevent the +# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex +import sys +if sys.version_info[0] == 2: + import unittest + import twisted.trial.unittest + if not getattr(unittest.TestCase, 'assertRaisesRegex', None): + unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None): + twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp diff --git a/tests/test_exporters.py b/tests/test_exporters.py index cd72c661a..0046c5666 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -103,7 +103,7 @@ class PythonItemExporterTest(BaseItemExporterTest): return PythonItemExporter(binary=False, **kwargs) def test_invalid_option(self): - with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"): + with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"): PythonItemExporter(invalid_option='something') def test_nested_item(self): diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 504937295..69d906fbf 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -147,11 +147,11 @@ class HeadersTest(unittest.TestCase): self.assertEqual(h1.getlist('hey'), [b'5']) def test_invalid_value(self): - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers, {'foo': object()}) - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers().__setitem__, 'foo', object()) - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers().setdefault, 'foo', object()) - self.assertRaisesRegexp(TypeError, 'Unsupported value type', - Headers().setlist, 'foo', [object()]) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers, {'foo': object()}) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers().__setitem__, 'foo', object()) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers().setdefault, 'foo', object()) + self.assertRaisesRegex(TypeError, 'Unsupported value type', + Headers().setlist, 'foo', [object()]) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 53d18d4aa..952e208de 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -989,9 +989,9 @@ class FormRequestTest(RequestTest): xpath = u"//form[@name='\u03b1']" encoded = xpath if six.PY3 else xpath.encode('unicode_escape') - self.assertRaisesRegexp(ValueError, re.escape(encoded), - self.request_class.from_response, - response, formxpath=xpath) + self.assertRaisesRegex(ValueError, re.escape(encoded), + self.request_class.from_response, + response, formxpath=xpath) def test_from_response_button_submit(self): response = _buildresponse( diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 3b90e3dac..cd5c3486e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -135,9 +135,9 @@ class BaseResponseTest(unittest.TestCase): r = self.response_class("http://example.com", body=b'hello') if self.response_class == Response: msg = "Response content isn't text" - self.assertRaisesRegexp(AttributeError, msg, getattr, r, 'text') - self.assertRaisesRegexp(NotSupported, msg, r.css, 'body') - self.assertRaisesRegexp(NotSupported, msg, r.xpath, '//body') + self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text') + self.assertRaisesRegex(NotSupported, msg, r.css, 'body') + self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body') else: r.text r.css('body') @@ -425,13 +425,13 @@ class TextResponseTest(BaseResponseTest): def test_follow_selector_list(self): resp = self._links_response() - self.assertRaisesRegexp(ValueError, 'SelectorList', - resp.follow, resp.css('a')) + self.assertRaisesRegex(ValueError, 'SelectorList', + resp.follow, resp.css('a')) def test_follow_selector_invalid(self): resp = self._links_response() - self.assertRaisesRegexp(ValueError, 'Unsupported', - resp.follow, resp.xpath('count(//div)')[0]) + self.assertRaisesRegex(ValueError, 'Unsupported', + resp.follow, resp.xpath('count(//div)')[0]) def test_follow_selector_attribute(self): resp = self._links_response() @@ -443,8 +443,8 @@ class TextResponseTest(BaseResponseTest): url='http://example.com', body=b'click me', ) - self.assertRaisesRegexp(ValueError, 'no href', - resp.follow, resp.css('a')[0]) + self.assertRaisesRegex(ValueError, 'no href', + resp.follow, resp.css('a')[0]) def test_follow_whitespace_selector(self): resp = self.response_class( diff --git a/tests/test_selector.py b/tests/test_selector.py index 210e3256c..b2565dd78 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -85,5 +85,5 @@ class SelectorTestCase(unittest.TestCase): x.__class__.__name__ def test_selector_bad_args(self): - with self.assertRaisesRegexp(ValueError, 'received both response and text'): + with self.assertRaisesRegex(ValueError, 'received both response and text'): Selector(TextResponse(url='http://example.com', body=b''), text=u'') diff --git a/tests/test_spider.py b/tests/test_spider.py index 1197ea653..e81e6d5f9 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -595,5 +595,5 @@ class NoParseMethodSpiderTest(unittest.TestCase): resp = TextResponse(url="http://www.example.com/random_url", body=text) exc_msg = 'Spider.parse callback is not defined' - with self.assertRaisesRegexp(NotImplementedError, exc_msg): + with self.assertRaisesRegex(NotImplementedError, exc_msg): spider.parse(resp) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 1cd59b99a..106da798c 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -84,8 +84,8 @@ class SpiderLoaderTest(unittest.TestCase): module = 'tests.test_spiderloader.test_spiders.spider1' runner = CrawlerRunner({'SPIDER_MODULES': [module]}) - self.assertRaisesRegexp(KeyError, 'Spider not found', - runner.create_crawler, 'spider2') + self.assertRaisesRegex(KeyError, 'Spider not found', + runner.create_crawler, 'spider2') crawler = runner.create_crawler('spider1') self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider)) From 27e63e6890896c86fa4b11dc9951c4773bc2d629 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 20:10:06 -0300 Subject: [PATCH 02/10] Monkey patch configparser module to ease the dropping of py2 support --- scrapy/_monkeypatches.py | 5 +++++ scrapy/utils/conf.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index f55ecc213..16bb4008b 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -14,6 +14,11 @@ if sys.version_info[0] == 2: from urlparse import uses_query uses_query.append('s3') + # Prevent the DeprecationWarning about SafeConfigParser -> ConfigParser + import configparser + if not getattr(configparser, 'ConfigParser', None): + configparser.ConfigParser = configparser.SafeConfigParser + # Undo what Twisted's perspective broker adds to pickle register # to prevent bugs like Twisted#7989 while serializing requests diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index fbd297340..26d66eaf8 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,10 +1,10 @@ import os import sys import numbers +import configparser from operator import itemgetter import six -from six.moves.configparser import SafeConfigParser from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath @@ -92,9 +92,9 @@ def init_env(project='default', set_syspath=True): def get_config(use_closest=True): - """Get Scrapy config file as a SafeConfigParser""" + """Get Scrapy config file as a ConfigParser""" sources = get_sources(use_closest) - cfg = SafeConfigParser() + cfg = configparser.ConfigParser() cfg.read(sources) return cfg From b714a372e21bed55d10539297bd90ef091f9ef43 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 20:23:10 -0300 Subject: [PATCH 03/10] [Tests] Prevent more DeprecationWarnings * assertRegexpMatches -> assertRegex * invalid escape sequence \[ --- tests/test_command_parse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index c18a6ce9f..98e415ad3 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -170,7 +170,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-c', 'dummy', self.url('/html')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find callback""", _textmode(stderr)) @defer.inlineCallbacks @@ -195,7 +195,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""No CrawlSpider rules found""", _textmode(stderr)) @defer.inlineCallbacks @@ -203,7 +203,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") @defer.inlineCallbacks def test_crawlspider_no_matching_rule(self): @@ -211,5 +211,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1} status, out, stderr = yield self.execute( ['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')] ) - self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""") + self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""") self.assertIn("""Cannot find a rule that matches""", _textmode(stderr)) From c24b80e1e6df7bd04cca77d068df2656d36a8753 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 20:34:31 -0300 Subject: [PATCH 04/10] Prevent DeprecationWarning about logging.warn --- scrapy/core/downloader/handlers/http11.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0673188a1..74ef82874 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -479,10 +479,10 @@ class _ResponseReader(protocol.Protocol): return elif not self._fail_on_dataloss_warned: - logger.warn("Got data loss in %s. If you want to process broken " - "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" - " -- This message won't be shown in further requests", - self._txresponse.request.absoluteURI.decode()) + logger.warning("Got data loss in %s. If you want to process broken " + "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" + " -- This message won't be shown in further requests", + self._txresponse.request.absoluteURI.decode()) self._fail_on_dataloss_warned = True self._finished.errback(reason) From 92d624c16189ced0d03eea8cf9db5fd400ebfabd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 20:55:45 -0300 Subject: [PATCH 05/10] [Tests] Prevent more TestCase DeprecationWarnings * assertRegexpMatches -> assertRegex --- tests/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/__init__.py b/tests/__init__.py index e55c77734..72ae12989 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -44,6 +44,8 @@ import sys if sys.version_info[0] == 2: import unittest import twisted.trial.unittest + if not getattr(unittest.TestCase, 'assertRegex', None): + unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches if not getattr(unittest.TestCase, 'assertRaisesRegex', None): unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None): From 40086dabb85a7c463f5a479e0c32d94c0b9463af Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 20:57:24 -0300 Subject: [PATCH 06/10] Prevent more DeprecationWarnings --- scrapy/core/downloader/handlers/ftp.py | 2 +- scrapy/core/downloader/handlers/http11.py | 2 +- scrapy/utils/template.py | 2 +- tests/test_downloadermiddleware_cookies.py | 2 +- tests/test_engine.py | 6 +++--- tests/test_linkextractors.py | 2 +- tests/test_loader.py | 2 +- tests/test_utils_url.py | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index c342d4ab1..806a537d4 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -59,7 +59,7 @@ class ReceivedDataProtocol(Protocol): def close(self): self.body.close() if self.filename else self.body.seek(0) -_CODE_RE = re.compile("\d+") +_CODE_RE = re.compile(r"\d+") class FTPDownloadHandler(object): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 74ef82874..cbaa36b2d 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -101,7 +101,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ - _responseMatcher = re.compile(b'HTTP/1\.. (?P\d{3})(?P.{,32})') + _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})') def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 1d7bd006c..615372fc8 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -18,7 +18,7 @@ def render_templatefile(path, **kwargs): os.remove(path) -CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]') +CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') def string_camelcase(string): """ Convert a word to its CamelCase version and remove invalid chars diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 17801e502..04884fb78 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -13,7 +13,7 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware class CookiesMiddlewareTest(TestCase): def assertCookieValEqual(self, first, second, msg=None): - cookievaleq = lambda cv: re.split(';\s*', cv.decode('latin1')) + cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1')) return self.assertEqual( sorted(cookievaleq(first)), sorted(cookievaleq(second)), msg) diff --git a/tests/test_engine.py b/tests/test_engine.py index 856465161..30150391a 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -40,9 +40,9 @@ class TestSpider(Spider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] - itemurl_re = re.compile("item\d+.html") - name_re = re.compile("

(.*?)

", re.M) - price_re = re.compile(">Price: \$(.*?)<", re.M) + itemurl_re = re.compile(r"item\d+.html") + name_re = re.compile(r"

(.*?)

", re.M) + price_re = re.compile(r">Price: \$(.*?)<", re.M) item_cls = TestItem diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index c9cd629f4..d96e259f6 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -288,7 +288,7 @@ class Base: response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') def process_value(value): - m = re.search("javascript:goToPage\('(.*?)'", value) + m = re.search(r"javascript:goToPage\('(.*?)'", value) if m: return m.group(1) diff --git a/tests/test_loader.py b/tests/test_loader.py index ce0fa0701..2725b001a 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -691,7 +691,7 @@ class SelectortemLoaderTest(unittest.TestCase): self.assertTrue(l.selector) l.add_css('url', 'a::attr(href)') self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org']) - l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)') + l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)') self.assertEqual(l.get_output_value('url'), [u'scrapy.org']) diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index c2b9fc176..e6588055c 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -233,8 +233,8 @@ for k, args in enumerate ([ setattr (GuessSchemeTest, t_method.__name__, t_method) # TODO: the following tests do not pass with current implementation -for k, args in enumerate ([ - ('C:\absolute\path\to\a\file.html', 'file://', +for k, args in enumerate([ + (r'C:\absolute\path\to\a\file.html', 'file://', 'Windows filepath are not supported for scrapy shell'), ], start=1): t_method = create_skipped_scheme_t(args) From eced544d64ac7398133781ecefed178ed5178892 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 22:14:47 -0300 Subject: [PATCH 07/10] Prevent even more DeprecationWarnings --- scrapy/item.py | 14 ++++++++---- scrapy/utils/datatypes.py | 13 ++++++++--- tests/test_downloadermiddleware_robotstxt.py | 23 +++++++++----------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index 031b80a2d..393302dce 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,16 +4,22 @@ Scrapy Item See documentation in docs/topics/item.rst """ -from pprint import pformat -from collections import MutableMapping -from copy import deepcopy - from abc import ABCMeta +from pprint import pformat +from copy import deepcopy +import collections + import six from scrapy.utils.trackref import object_ref +if six.PY3: + MutableMapping = collections.abc.MutableMapping +else: + MutableMapping = collections.MutableMapping + + class BaseItem(object_ref): """Base class for all scraped items.""" pass diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index eb373c501..af39e887f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,13 +6,20 @@ This module must not depend on any module outside the Standard Library. """ import copy -import six +import collections import warnings -from collections import OrderedDict, Mapping + +import six from scrapy.exceptions import ScrapyDeprecationWarning +if six.PY3: + Mapping = collections.abc.Mapping +else: + Mapping = collections.Mapping + + class MultiValueDictKeyError(KeyError): def __init__(self, *args, **kwargs): warnings.warn( @@ -289,7 +296,7 @@ class MergeDict(object): return self.__copy__() -class LocalCache(OrderedDict): +class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. Older items expires first. diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 60306eacb..2b3548bdd 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import re from twisted.internet import reactor, error from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -31,18 +30,16 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = re.sub(b'^\s+(?m)', b'', u''' - User-Agent: * - Disallow: /admin/ - Disallow: /static/ - - # taken from https://en.wikipedia.org/robots.txt - Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: - Disallow: /wiki/Käyttäjä: - - User-Agent: UnicödeBöt - Disallow: /some/randome/page.html - '''.encode('utf-8')) + ROBOTS = u""" +User-Agent: * +Disallow: /admin/ +Disallow: /static/ +# taken from https://en.wikipedia.org/robots.txt +Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: +Disallow: /wiki/Käyttäjä: +User-Agent: UnicödeBöt +Disallow: /some/randome/page.html +""".encode('utf-8') response = TextResponse('http://site.local/robots.txt', body=ROBOTS) def return_response(request, spider): deferred = Deferred() From ef9a61921482f1a0004616e0b2b6734b9314655d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 23:36:39 -0300 Subject: [PATCH 08/10] More DeprecationWarnings --- scrapy/settings/__init__.py | 10 ++++++++-- tests/test_utils_datatypes.py | 9 ++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 14c93bef2..6b53c7a79 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,11 +1,17 @@ import six import json import copy -from collections import MutableMapping +import collections from importlib import import_module from pprint import pformat -from . import default_settings +from scrapy.settings import default_settings + + +if six.PY3: + MutableMapping = collections.abc.MutableMapping +else: + MutableMapping = collections.MutableMapping SETTINGS_PRIORITIES = { diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 5b83869b8..618050fef 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,11 +1,18 @@ import copy import unittest -from collections import Mapping, MutableMapping + +import six +if six.PY3: + from collections.abc import Mapping, MutableMapping +else: + from collections.abc import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, SequenceExclude + __doctests__ = ['scrapy.utils.datatypes'] + class CaselessDictTest(unittest.TestCase): def test_init_dict(self): From d5a2a7032979c6af8b074ec7fd60baa5256cc7cd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 13 Jul 2019 23:47:41 -0300 Subject: [PATCH 09/10] Fix import --- tests/test_utils_datatypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 618050fef..0d6e1eb71 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -5,7 +5,7 @@ import six if six.PY3: from collections.abc import Mapping, MutableMapping else: - from collections.abc import Mapping, MutableMapping + from collections import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, SequenceExclude From 7e3a602d569bd81e9bd891a482f440a93dbe7bb4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 15 Jul 2019 12:37:09 -0300 Subject: [PATCH 10/10] Suggested changes --- scrapy/_monkeypatches.py | 10 +++------- scrapy/item.py | 6 +++--- scrapy/settings/__init__.py | 6 +++--- scrapy/utils/datatypes.py | 6 +++--- tests/__init__.py | 4 ++-- tests/test_utils_datatypes.py | 6 +++--- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 16bb4008b..935c4bfa3 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,7 +1,8 @@ -import sys +import six from six.moves import copyreg -if sys.version_info[0] == 2: + +if six.PY2: from urlparse import urlparse # workaround for https://bugs.python.org/issue7904 - Python < 2.7 @@ -14,11 +15,6 @@ if sys.version_info[0] == 2: from urlparse import uses_query uses_query.append('s3') - # Prevent the DeprecationWarning about SafeConfigParser -> ConfigParser - import configparser - if not getattr(configparser, 'ConfigParser', None): - configparser.ConfigParser = configparser.SafeConfigParser - # Undo what Twisted's perspective broker adds to pickle register # to prevent bugs like Twisted#7989 while serializing requests diff --git a/scrapy/item.py b/scrapy/item.py index 393302dce..9d4786788 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -14,10 +14,10 @@ import six from scrapy.utils.trackref import object_ref -if six.PY3: - MutableMapping = collections.abc.MutableMapping -else: +if six.PY2: MutableMapping = collections.MutableMapping +else: + MutableMapping = collections.abc.MutableMapping class BaseItem(object_ref): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 6b53c7a79..f28c7940d 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -8,10 +8,10 @@ from pprint import pformat from scrapy.settings import default_settings -if six.PY3: - MutableMapping = collections.abc.MutableMapping -else: +if six.PY2: MutableMapping = collections.MutableMapping +else: + MutableMapping = collections.abc.MutableMapping SETTINGS_PRIORITIES = { diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index af39e887f..b98a1297e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -14,10 +14,10 @@ import six from scrapy.exceptions import ScrapyDeprecationWarning -if six.PY3: - Mapping = collections.abc.Mapping -else: +if six.PY2: Mapping = collections.Mapping +else: + Mapping = collections.abc.Mapping class MultiValueDictKeyError(KeyError): diff --git a/tests/__init__.py b/tests/__init__.py index 72ae12989..9c9e35c35 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -40,8 +40,8 @@ def get_testdata(*paths): # FIXME: delete after dropping py2 support # Monkey patch the unittest module to prevent the # DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex -import sys -if sys.version_info[0] == 2: +import six +if six.PY2: import unittest import twisted.trial.unittest if not getattr(unittest.TestCase, 'assertRegex', None): diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 0d6e1eb71..535095b8d 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -2,10 +2,10 @@ import copy import unittest import six -if six.PY3: - from collections.abc import Mapping, MutableMapping -else: +if six.PY2: from collections import Mapping, MutableMapping +else: + from collections.abc import Mapping, MutableMapping from scrapy.utils.datatypes import CaselessDict, SequenceExclude