RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3877 from elacuesta/tests_deprecation_warnings 

Prevent deprecation warnings
This commit is contained in:
Mikhail Korobov 2019-07-16 21:58:44 +05:00 committed by GitHub
parent fa6a0d799b 7e3a602d56
commit 377d8a7be7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 111 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/_monkeypatches.py
View File

@ -1,7 +1,8 @@
import sys
import six
from six.moves import copyreg
if sys.version_info[0] == 2:
if six.PY2:
from urlparse import urlparse
# workaround for https://bugs.python.org/issue7904 - Python < 2.7

2
scrapy/core/downloader/handlers/ftp.py
View File

@ -59,7 +59,7 @@ class ReceivedDataProtocol(Protocol):
def close(self):
self.body.close() if self.filename else self.body.seek(0)
_CODE_RE = re.compile("\d+")
_CODE_RE = re.compile(r"\d+")
class FTPDownloadHandler(object):

10
scrapy/core/downloader/handlers/http11.py
View File

@ -101,7 +101,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
for it.
"""
_responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
_responseMatcher = re.compile(br'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
def __init__(self, reactor, host, port, proxyConf, contextFactory,
timeout=30, bindAddress=None):
@ -479,10 +479,10 @@ class _ResponseReader(protocol.Protocol):
return
elif not self._fail_on_dataloss_warned:
logger.warn("Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",
self._txresponse.request.absoluteURI.decode())
logger.warning("Got data loss in %s. If you want to process broken "
"responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False"
" -- This message won't be shown in further requests",
self._txresponse.request.absoluteURI.decode())
self._fail_on_dataloss_warned = True
self._finished.errback(reason)

14
scrapy/item.py
View File

@ -4,16 +4,22 @@ Scrapy Item
See documentation in docs/topics/item.rst
"""
from pprint import pformat
from collections import MutableMapping
from copy import deepcopy
from abc import ABCMeta
from pprint import pformat
from copy import deepcopy
import collections
import six
from scrapy.utils.trackref import object_ref
if six.PY2:
MutableMapping = collections.MutableMapping
else:
MutableMapping = collections.abc.MutableMapping
class BaseItem(object_ref):
"""Base class for all scraped items."""
pass

10
scrapy/settings/__init__.py
View File

@ -1,11 +1,17 @@
import six
import json
import copy
from collections import MutableMapping
import collections
from importlib import import_module
from pprint import pformat
from . import default_settings
from scrapy.settings import default_settings
if six.PY2:
MutableMapping = collections.MutableMapping
else:
MutableMapping = collections.abc.MutableMapping
SETTINGS_PRIORITIES = {

6
scrapy/utils/conf.py
View File

@ -1,10 +1,10 @@
import os
import sys
import numbers
import configparser
from operator import itemgetter
import six
from six.moves.configparser import SafeConfigParser
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
@ -92,9 +92,9 @@ def init_env(project='default', set_syspath=True):
def get_config(use_closest=True):
"""Get Scrapy config file as a SafeConfigParser"""
"""Get Scrapy config file as a ConfigParser"""
sources = get_sources(use_closest)
cfg = SafeConfigParser()
cfg = configparser.ConfigParser()
cfg.read(sources)
return cfg

13
scrapy/utils/datatypes.py
View File

@ -6,13 +6,20 @@ This module must not depend on any module outside the Standard Library.
"""
import copy
import six
import collections
import warnings
from collections import OrderedDict, Mapping
import six
from scrapy.exceptions import ScrapyDeprecationWarning
if six.PY2:
Mapping = collections.Mapping
else:
Mapping = collections.abc.Mapping
class MultiValueDictKeyError(KeyError):
def __init__(self, *args, **kwargs):
warnings.warn(
@ -289,7 +296,7 @@ class MergeDict(object):
return self.__copy__()
class LocalCache(OrderedDict):
class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.
Older items expires first.

2
scrapy/utils/template.py
View File

@ -18,7 +18,7 @@ def render_templatefile(path, **kwargs):
os.remove(path)
CAMELCASE_INVALID_CHARS = re.compile('[^a-zA-Z\d]')
CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]')
def string_camelcase(string):
""" Convert a word to its CamelCase version and remove invalid chars

15
tests/__init__.py
View File

@ -35,3 +35,18 @@ def get_testdata(*paths):
path = os.path.join(tests_datadir, *paths)
with open(path, 'rb') as f:
return f.read()
# FIXME: delete after dropping py2 support
# Monkey patch the unittest module to prevent the
# DeprecationWarning about assertRaisesRegexp -> assertRaisesRegex
import six
if six.PY2:
import unittest
import twisted.trial.unittest
if not getattr(unittest.TestCase, 'assertRegex', None):
unittest.TestCase.assertRegex = unittest.TestCase.assertRegexpMatches
if not getattr(unittest.TestCase, 'assertRaisesRegex', None):
unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
if not getattr(twisted.trial.unittest.TestCase, 'assertRaisesRegex', None):
twisted.trial.unittest.TestCase.assertRaisesRegex = twisted.trial.unittest.TestCase.assertRaisesRegexp

8
tests/test_command_parse.py
View File

@ -170,7 +170,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-c', 'dummy', self.url('/html')]
)
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find callback""", _textmode(stderr))
@defer.inlineCallbacks
@ -195,7 +195,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', self.spider_name, '-r', self.url('/html')]
)
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
self.assertIn("""No CrawlSpider rules found""", _textmode(stderr))
@defer.inlineCallbacks
@ -203,7 +203,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
)
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
@defer.inlineCallbacks
def test_crawlspider_no_matching_rule(self):
@ -211,5 +211,5 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
status, out, stderr = yield self.execute(
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
)
self.assertRegexpMatches(_textmode(out), """# Scraped Items -+\n\[\]""")
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))

2
tests/test_downloadermiddleware_cookies.py
View File

@ -13,7 +13,7 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
class CookiesMiddlewareTest(TestCase):
def assertCookieValEqual(self, first, second, msg=None):
cookievaleq = lambda cv: re.split(';\s*', cv.decode('latin1'))
cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1'))
return self.assertEqual(
sorted(cookievaleq(first)),
sorted(cookievaleq(second)), msg)

23
tests/test_downloadermiddleware_robotstxt.py
View File

@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import re
from twisted.internet import reactor, error
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python import failure
@ -31,18 +30,16 @@ class RobotsTxtMiddlewareTest(unittest.TestCase):
def _get_successful_crawler(self):
crawler = self.crawler
crawler.settings.set('ROBOTSTXT_OBEY', True)
ROBOTS = re.sub(b'^\s+(?m)', b'', u'''
User-Agent: *
Disallow: /admin/
Disallow: /static/
# taken from https://en.wikipedia.org/robots.txt
Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
Disallow: /wiki/Käyttäjä:
User-Agent: UnicödeBöt
Disallow: /some/randome/page.html
'''.encode('utf-8'))
ROBOTS = u"""
User-Agent: *
Disallow: /admin/
Disallow: /static/
# taken from https://en.wikipedia.org/robots.txt
Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
Disallow: /wiki/Käyttäjä:
User-Agent: UnicödeBöt
Disallow: /some/randome/page.html
""".encode('utf-8')
response = TextResponse('http://site.local/robots.txt', body=ROBOTS)
def return_response(request, spider):
deferred = Deferred()

6
tests/test_engine.py
View File

@ -40,9 +40,9 @@ class TestSpider(Spider):
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
itemurl_re = re.compile("item\d+.html")
name_re = re.compile("<h1>(.*?)</h1>", re.M)
price_re = re.compile(">Price: \$(.*?)<", re.M)
itemurl_re = re.compile(r"item\d+.html")
name_re = re.compile(r"<h1>(.*?)</h1>", re.M)
price_re = re.compile(r">Price: \$(.*?)<", re.M)
item_cls = TestItem

2
tests/test_exporters.py
View File

@ -103,7 +103,7 @@ class PythonItemExporterTest(BaseItemExporterTest):
return PythonItemExporter(binary=False, **kwargs)
def test_invalid_option(self):
with self.assertRaisesRegexp(TypeError, "Unexpected options: invalid_option"):
with self.assertRaisesRegex(TypeError, "Unexpected options: invalid_option"):
PythonItemExporter(invalid_option='something')
def test_nested_item(self):

16
tests/test_http_headers.py
View File

@ -147,11 +147,11 @@ class HeadersTest(unittest.TestCase):
self.assertEqual(h1.getlist('hey'), [b'5'])
def test_invalid_value(self):
self.assertRaisesRegexp(TypeError, 'Unsupported value type',
Headers, {'foo': object()})
self.assertRaisesRegexp(TypeError, 'Unsupported value type',
Headers().__setitem__, 'foo', object())
self.assertRaisesRegexp(TypeError, 'Unsupported value type',
Headers().setdefault, 'foo', object())
self.assertRaisesRegexp(TypeError, 'Unsupported value type',
Headers().setlist, 'foo', [object()])
self.assertRaisesRegex(TypeError, 'Unsupported value type',
Headers, {'foo': object()})
self.assertRaisesRegex(TypeError, 'Unsupported value type',
Headers().__setitem__, 'foo', object())
self.assertRaisesRegex(TypeError, 'Unsupported value type',
Headers().setdefault, 'foo', object())
self.assertRaisesRegex(TypeError, 'Unsupported value type',
Headers().setlist, 'foo', [object()])

6
tests/test_http_request.py
View File

@ -989,9 +989,9 @@ class FormRequestTest(RequestTest):
xpath = u"//form[@name='\u03b1']"
encoded = xpath if six.PY3 else xpath.encode('unicode_escape')
self.assertRaisesRegexp(ValueError, re.escape(encoded),
self.request_class.from_response,
response, formxpath=xpath)
self.assertRaisesRegex(ValueError, re.escape(encoded),
self.request_class.from_response,
response, formxpath=xpath)
def test_from_response_button_submit(self):
response = _buildresponse(

18
tests/test_http_response.py
View File

@ -135,9 +135,9 @@ class BaseResponseTest(unittest.TestCase):
r = self.response_class("http://example.com", body=b'hello')
if self.response_class == Response:
msg = "Response content isn't text"
self.assertRaisesRegexp(AttributeError, msg, getattr, r, 'text')
self.assertRaisesRegexp(NotSupported, msg, r.css, 'body')
self.assertRaisesRegexp(NotSupported, msg, r.xpath, '//body')
self.assertRaisesRegex(AttributeError, msg, getattr, r, 'text')
self.assertRaisesRegex(NotSupported, msg, r.css, 'body')
self.assertRaisesRegex(NotSupported, msg, r.xpath, '//body')
else:
r.text
r.css('body')
@ -425,13 +425,13 @@ class TextResponseTest(BaseResponseTest):
def test_follow_selector_list(self):
resp = self._links_response()
self.assertRaisesRegexp(ValueError, 'SelectorList',
resp.follow, resp.css('a'))
self.assertRaisesRegex(ValueError, 'SelectorList',
resp.follow, resp.css('a'))
def test_follow_selector_invalid(self):
resp = self._links_response()
self.assertRaisesRegexp(ValueError, 'Unsupported',
resp.follow, resp.xpath('count(//div)')[0])
self.assertRaisesRegex(ValueError, 'Unsupported',
resp.follow, resp.xpath('count(//div)')[0])
def test_follow_selector_attribute(self):
resp = self._links_response()
@ -443,8 +443,8 @@ class TextResponseTest(BaseResponseTest):
url='http://example.com',
body=b'<html><body><a name=123>click me</a></body></html>',
)
self.assertRaisesRegexp(ValueError, 'no href',
resp.follow, resp.css('a')[0])
self.assertRaisesRegex(ValueError, 'no href',
resp.follow, resp.css('a')[0])
def test_follow_whitespace_selector(self):
resp = self.response_class(

2
tests/test_linkextractors.py
View File

@ -288,7 +288,7 @@ class Base:
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
def process_value(value):
m = re.search("javascript:goToPage\('(.*?)'", value)
m = re.search(r"javascript:goToPage\('(.*?)'", value)
if m:
return m.group(1)

2
tests/test_loader.py
View File

@ -691,7 +691,7 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertTrue(l.selector)
l.add_css('url', 'a::attr(href)')
self.assertEqual(l.get_output_value('url'), [u'http://www.scrapy.org'])
l.replace_css('url', 'a::attr(href)', re='http://www\.(.+)')
l.replace_css('url', 'a::attr(href)', re=r'http://www\.(.+)')
self.assertEqual(l.get_output_value('url'), [u'scrapy.org'])

2
tests/test_selector.py
View File

@ -85,5 +85,5 @@ class SelectorTestCase(unittest.TestCase):
x.__class__.__name__
def test_selector_bad_args(self):
with self.assertRaisesRegexp(ValueError, 'received both response and text'):
with self.assertRaisesRegex(ValueError, 'received both response and text'):
Selector(TextResponse(url='http://example.com', body=b''), text=u'')

2
tests/test_spider.py
View File

@ -595,5 +595,5 @@ class NoParseMethodSpiderTest(unittest.TestCase):
resp = TextResponse(url="http://www.example.com/random_url", body=text)
exc_msg = 'Spider.parse callback is not defined'
with self.assertRaisesRegexp(NotImplementedError, exc_msg):
with self.assertRaisesRegex(NotImplementedError, exc_msg):
spider.parse(resp)

4
tests/test_spiderloader/__init__.py
View File

@ -84,8 +84,8 @@ class SpiderLoaderTest(unittest.TestCase):
module = 'tests.test_spiderloader.test_spiders.spider1'
runner = CrawlerRunner({'SPIDER_MODULES': [module]})
self.assertRaisesRegexp(KeyError, 'Spider not found',
runner.create_crawler, 'spider2')
self.assertRaisesRegex(KeyError, 'Spider not found',
runner.create_crawler, 'spider2')
crawler = runner.create_crawler('spider1')
self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))

9
tests/test_utils_datatypes.py
View File

@ -1,11 +1,18 @@
import copy
import unittest
from collections import Mapping, MutableMapping
import six
if six.PY2:
from collections import Mapping, MutableMapping
else:
from collections.abc import Mapping, MutableMapping
from scrapy.utils.datatypes import CaselessDict, SequenceExclude
__doctests__ = ['scrapy.utils.datatypes']
class CaselessDictTest(unittest.TestCase):
def test_init_dict(self):

4
tests/test_utils_url.py
View File

@ -233,8 +233,8 @@ for k, args in enumerate ([
setattr (GuessSchemeTest, t_method.__name__, t_method)
# TODO: the following tests do not pass with current implementation
for k, args in enumerate ([
('C:\absolute\path\to\a\file.html', 'file://',
for k, args in enumerate([
(r'C:\absolute\path\to\a\file.html', 'file://',
'Windows filepath are not supported for scrapy shell'),
], start=1):
t_method = create_skipped_scheme_t(args)