RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change super syntax (#4707)

This commit is contained in:
Marc Hernández 2020-08-04 20:42:01 +02:00 committed by GitHub
parent 4ee538e44b
commit 336f19f5cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
56 changed files with 117 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extras/qpsclient.py
View File

@ -27,7 +27,7 @@ class QPSSpider(Spider):
slots = 1
def __init__(self, *a, **kw):
super(QPSSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps

2
scrapy/commands/view.py
View File

@ -11,7 +11,7 @@ class Command(fetch.Command):
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
def add_options(self, parser):
super(Command, self).add_options(parser)
super().add_options(parser)
parser.remove_option("--headers")
def _print_response(self, response, opts):

2
scrapy/contracts/default.py
View File

@ -56,7 +56,7 @@ class ReturnsContract(Contract):
}
def __init__(self, *args, **kwargs):
super(ReturnsContract, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
if len(self.args) not in [1, 2, 3]:
raise ValueError(

4
scrapy/core/downloader/contextfactory.py
View File

@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
if tls_ciphers:
@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
#
# * getattr() for `_ssl_method` attribute for context factories
# not calling super(..., self).__init__
# not calling super().__init__
return CertificateOptions(
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),

10
scrapy/core/downloader/handlers/http11.py
View File

@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
self._tunnelReadyDeferred = defer.Deferred()
self._tunneledHost = host
self._tunneledPort = port
@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def connect(self, protocolFactory):
self._protocolFactory = protocolFactory
connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
connectDeferred = super().connect(protocolFactory)
connectDeferred.addCallback(self.requestTunnel)
connectDeferred.addErrback(self.connectFailed)
return self._tunnelReadyDeferred
@ -215,7 +215,7 @@ class TunnelingAgent(Agent):
def __init__(self, reactor, proxyConf, contextFactory=None,
connectTimeout=None, bindAddress=None, pool=None):
super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
self._proxyConf = proxyConf
self._contextFactory = contextFactory
@ -235,7 +235,7 @@ class TunnelingAgent(Agent):
# otherwise, same remote host connection request could reuse
# a cached tunneled connection to a different proxy
key = key + self._proxyConf
return super(TunnelingAgent, self)._requestWithEndpoint(
return super()._requestWithEndpoint(
key=key,
endpoint=endpoint,
method=method,
@ -249,7 +249,7 @@ class TunnelingAgent(Agent):
class ScrapyProxyAgent(Agent):
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
super(ScrapyProxyAgent, self).__init__(
super().__init__(
reactor=reactor,
connectTimeout=connectTimeout,
bindAddress=bindAddress,

2
scrapy/core/downloader/tls.py
View File

@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
"""
def __init__(self, hostname, ctx, verbose_logging=False):
super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
super().__init__(hostname, ctx)
self.verbose_logging = verbose_logging
def _identityVerifyingInfoCallback(self, connection, where, ret):

2
scrapy/core/spidermw.py
View File

@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
def _add_middleware(self, mw):
super(SpiderMiddlewareManager, self)._add_middleware(mw)
super()._add_middleware(mw)
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_start_requests'):

2
scrapy/crawler.py
View File

@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
"""
def __init__(self, settings=None, install_root_handler=True):
super(CrawlerProcess, self).__init__(settings)
super().__init__(settings)
install_shutdown_handlers(self._signal_shutdown)
configure_logging(self.settings, install_root_handler)
log_scrapy_info(self.settings)

2
scrapy/downloadermiddlewares/redirect.py
View File

@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
enabled_setting = 'METAREFRESH_ENABLED'
def __init__(self, settings):
super(MetaRefreshMiddleware, self).__init__(settings)
super().__init__(settings)
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')

4
scrapy/exceptions.py
View File

@ -37,7 +37,7 @@ class CloseSpider(Exception):
"""Raise this from callbacks to request the spider to be closed"""
def __init__(self, reason='cancelled'):
super(CloseSpider, self).__init__()
super().__init__()
self.reason = reason
@ -74,7 +74,7 @@ class UsageError(Exception):
def __init__(self, *a, **kw):
self.print_help = kw.pop('print_help', True)
super(UsageError, self).__init__(*a, **kw)
super().__init__(*a, **kw)
class ScrapyDeprecationWarning(Warning):

2
scrapy/exporters.py
View File

@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter):
def _configure(self, options, dont_fail=False):
self.binary = options.pop('binary', True)
super(PythonItemExporter, self)._configure(options, dont_fail)
super()._configure(options, dont_fail)
if self.binary:
warnings.warn(
"PythonItemExporter will drop support for binary export in the future",

8
scrapy/http/headers.py
View File

@ -8,7 +8,7 @@ class Headers(CaselessDict):
def __init__(self, seq=None, encoding='utf-8'):
self.encoding = encoding
super(Headers, self).__init__(seq)
super().__init__(seq)
def normkey(self, key):
"""Normalize key to bytes"""
@ -37,19 +37,19 @@ class Headers(CaselessDict):
def __getitem__(self, key):
try:
return super(Headers, self).__getitem__(key)[-1]
return super().__getitem__(key)[-1]
except IndexError:
return None
def get(self, key, def_val=None):
try:
return super(Headers, self).get(key, def_val)[-1]
return super().get(key, def_val)[-1]
except IndexError:
return None
def getlist(self, key, def_val=None):
try:
return super(Headers, self).__getitem__(key)
return super().__getitem__(key)
except KeyError:
if def_val is not None:
return self.normvalue(def_val)

2
scrapy/http/request/form.py
View File

@ -24,7 +24,7 @@ class FormRequest(Request):
if formdata and kwargs.get('method') is None:
kwargs['method'] = 'POST'
super(FormRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
if formdata:
items = formdata.items() if isinstance(formdata, dict) else formdata

4
scrapy/http/request/json_request.py
View File

@ -32,7 +32,7 @@ class JsonRequest(Request):
if 'method' not in kwargs:
kwargs['method'] = 'POST'
super(JsonRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'application/json')
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
@ -47,7 +47,7 @@ class JsonRequest(Request):
elif not body_passed and data_passed:
kwargs['body'] = self._dumps(data)
return super(JsonRequest, self).replace(*args, **kwargs)
return super().replace(*args, **kwargs)
def _dumps(self, data):
"""Convert to JSON """

2
scrapy/http/request/rpc.py
View File

@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
if encoding is not None:
kwargs['encoding'] = encoding
super(XmlRpcRequest, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.headers.setdefault('Content-Type', 'text/xml')

10
scrapy/http/response/text.py
View File

@ -35,13 +35,13 @@ class TextResponse(Response):
self._cached_benc = None
self._cached_ubody = None
self._cached_selector = None
super(TextResponse, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
def _set_url(self, url):
if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
super()._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
@ -51,7 +51,7 @@ class TextResponse(Response):
type(self).__name__)
self._body = body.encode(self._encoding)
else:
super(TextResponse, self)._set_body(body)
super()._set_body(body)
def replace(self, *args, **kwargs):
kwargs.setdefault('encoding', self.encoding)
@ -166,7 +166,7 @@ class TextResponse(Response):
elif isinstance(url, parsel.SelectorList):
raise ValueError("SelectorList is not supported")
encoding = self.encoding if encoding is None else encoding
return super(TextResponse, self).follow(
return super().follow(
url=url,
callback=callback,
method=method,
@ -226,7 +226,7 @@ class TextResponse(Response):
for sel in selectors:
with suppress(_InvalidSelector):
urls.append(_url_from_selector(sel))
return super(TextResponse, self).follow_all(
return super().follow_all(
urls=urls,
callback=callback,
method=method,

10
scrapy/item.py
View File

@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(BaseItem, cls).__new__(cls, *args, **kwargs)
return super().__new__(cls, *args, **kwargs)
class Field(dict):
@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
def __new__(mcs, class_name, bases, attrs):
classcell = attrs.pop('__classcell__', None)
new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
_class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
_class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
fields = getattr(_class, 'fields', {})
new_attrs = {}
@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
new_attrs['_class'] = _class
if classcell is not None:
new_attrs['__classcell__'] = classcell
return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
return super().__new__(mcs, class_name, bases, new_attrs)
class DictItem(MutableMapping, BaseItem):
@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
if issubclass(cls, DictItem) and not issubclass(cls, Item):
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(DictItem, cls).__new__(cls, *args, **kwargs)
return super().__new__(cls, *args, **kwargs)
def __init__(self, *args, **kwargs):
self._values = {}
@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):
def __setattr__(self, name, value):
if not name.startswith('_'):
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
super(DictItem, self).__setattr__(name, value)
super().__setattr__(name, value)
def __len__(self):
return len(self._values)

2
scrapy/linkextractors/__init__.py
View File

@ -65,7 +65,7 @@ class FilteringLinkExtractor:
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)
return super(FilteringLinkExtractor, cls).__new__(cls)
return super().__new__(cls)
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):

2
scrapy/linkextractors/lxmlhtml.py
View File

@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
strip=strip,
canonicalized=canonicalize
)
super(LxmlLinkExtractor, self).__init__(
super().__init__(
link_extractor=lx,
allow=allow,
deny=deny,

2
scrapy/pipelines/files.py
View File

@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):
resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
)
super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
super().__init__(download_func=download_func, settings=settings)
@classmethod
def from_settings(cls, settings):

3
scrapy/pipelines/images.py
View File

@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):
DEFAULT_IMAGES_RESULT_FIELD = 'images'
def __init__(self, store_uri, download_func=None, settings=None):
super(ImagesPipeline, self).__init__(store_uri, settings=settings,
download_func=download_func)
super().__init__(store_uri, settings=settings, download_func=download_func)
if isinstance(settings, dict) or settings is None:
settings = Settings(settings)

10
scrapy/resolver.py
View File

@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):
"""
def __init__(self, reactor, cache_size, timeout):
super(CachingThreadedResolver, self).__init__(reactor)
super().__init__(reactor)
dnscache.limit = cache_size
self.timeout = timeout
@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):
# so the input argument above is simply overridden
# to enforce Scrapy's DNS_TIMEOUT setting's value
timeout = (self.timeout,)
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
d = super().getHostByName(name, timeout)
if dnscache.limit:
d.addCallback(self._cache_result, name)
return d
@ -80,16 +80,16 @@ class CachingHostnameResolver:
class CachingResolutionReceiver(resolutionReceiver):
def resolutionBegan(self, resolution):
super(CachingResolutionReceiver, self).resolutionBegan(resolution)
super().resolutionBegan(resolution)
self.resolution = resolution
self.resolved = False
def addressResolved(self, address):
super(CachingResolutionReceiver, self).addressResolved(address)
super().addressResolved(address)
self.resolved = True
def resolutionComplete(self):
super(CachingResolutionReceiver, self).resolutionComplete()
super().resolutionComplete()
if self.resolved:
dnscache[hostName] = self.resolution

2
scrapy/selector/unified.py
View File

@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):
kwargs.setdefault('base_url', response.url)
self.response = response
super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
super().__init__(text=text, type=st, root=root, **kwargs)

2
scrapy/settings/__init__.py
View File

@ -439,7 +439,7 @@ class Settings(BaseSettings):
# Do not pass kwarg values here. We don't want to promote user-defined
# dicts, and we want to update, not replace, default dicts with the
# values given by the user
super(Settings, self).__init__()
super().__init__()
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities

2
scrapy/spidermiddlewares/httperror.py
View File

@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):
def __init__(self, response, *args, **kwargs):
self.response = response
super(HttpError, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
class HttpErrorMiddleware:

4
scrapy/spiders/crawl.py
View File

@ -75,7 +75,7 @@ class CrawlSpider(Spider):
rules = ()
def __init__(self, *a, **kw):
super(CrawlSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
self._compile_rules()
def _parse(self, response, **kwargs):
@ -145,6 +145,6 @@ class CrawlSpider(Spider):
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
spider = super().from_crawler(crawler, *args, **kwargs)
spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
return spider

2
scrapy/spiders/init.py
View File

@ -6,7 +6,7 @@ class InitSpider(Spider):
"""Base Spider with initialization facilities"""
def start_requests(self):
self._postinit_reqs = super(InitSpider, self).start_requests()
self._postinit_reqs = super().start_requests()
return iterate_spider_output(self.init_request())
def initialized(self, response=None):

2
scrapy/spiders/sitemap.py
View File

@ -18,7 +18,7 @@ class SitemapSpider(Spider):
sitemap_alternate_links = False
def __init__(self, *a, **kw):
super(SitemapSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
if isinstance(c, str):

12
scrapy/squeues.py
View File

@ -20,7 +20,7 @@ def _with_mkdir(queue_class):
if not os.path.exists(dirname):
os.makedirs(dirname, exist_ok=True)
super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
super().__init__(path, *args, **kwargs)
return DirectoriesCreated
@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
def push(self, obj):
s = serialize(obj)
super(SerializableQueue, self).push(s)
super().push(s)
def pop(self):
s = super(SerializableQueue, self).pop()
s = super().pop()
if s:
return deserialize(s)
@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
def __init__(self, crawler, key):
self.spider = crawler.spider
super(ScrapyRequestQueue, self).__init__(key)
super().__init__(key)
@classmethod
def from_crawler(cls, crawler, key, *args, **kwargs):
@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):
def push(self, request):
request = request_to_dict(request, self.spider)
return super(ScrapyRequestQueue, self).push(request)
return super().push(request)
def pop(self):
request = super(ScrapyRequestQueue, self).pop()
request = super().pop()
if not request:
return None

2
scrapy/statscollectors.py
View File

@ -54,7 +54,7 @@ class StatsCollector:
class MemoryStatsCollector(StatsCollector):
def __init__(self, crawler):
super(MemoryStatsCollector, self).__init__(crawler)
super().__init__(crawler)
self.spider_stats = {}
def _persist_stats(self, stats, spider):

14
scrapy/utils/datatypes.py
View File

@ -15,7 +15,7 @@ class CaselessDict(dict):
__slots__ = ()
def __init__(self, seq=None):
super(CaselessDict, self).__init__()
super().__init__()
if seq:
self.update(seq)
@ -53,7 +53,7 @@ class CaselessDict(dict):
def update(self, seq):
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super(CaselessDict, self).update(iseq)
super().update(iseq)
@classmethod
def fromkeys(cls, keys, value=None):
@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):
"""
def __init__(self, limit=None):
super(LocalCache, self).__init__()
super().__init__()
self.limit = limit
def __setitem__(self, key, value):
if self.limit:
while len(self) >= self.limit:
self.popitem(last=False)
super(LocalCache, self).__setitem__(key, value)
super().__setitem__(key, value)
class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
"""
def __init__(self, limit=None):
super(LocalWeakReferencedCache, self).__init__()
super().__init__()
self.data = LocalCache(limit=limit)
def __setitem__(self, key, value):
try:
super(LocalWeakReferencedCache, self).__setitem__(key, value)
super().__setitem__(key, value)
except TypeError:
pass # key is not weak-referenceable, skip caching
def __getitem__(self, key):
try:
return super(LocalWeakReferencedCache, self).__getitem__(key)
return super().__getitem__(key)
except (TypeError, KeyError):
return None # key is either not weak-referenceable or not cached

8
scrapy/utils/deprecate.py
View File

@ -57,7 +57,7 @@ def create_deprecated_class(
warned_on_subclass = False
def __new__(metacls, name, bases, clsdict_):
cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
cls = super().__new__(metacls, name, bases, clsdict_)
if metacls.deprecated_class is None:
metacls.deprecated_class = cls
return cls
@ -73,7 +73,7 @@ def create_deprecated_class(
if warn_once:
msg += ' (warning only on first subclass, there may be others)'
warnings.warn(msg, warn_category, stacklevel=2)
super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
super().__init__(name, bases, clsdict_)
# see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
# and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
@ -88,7 +88,7 @@ def create_deprecated_class(
# is the deprecated class itself - subclasses of the
# deprecated class should not use custom `__subclasscheck__`
# method.
return super(DeprecatedClass, cls).__subclasscheck__(sub)
return super().__subclasscheck__(sub)
if not inspect.isclass(sub):
raise TypeError("issubclass() arg 1 must be a class")
@ -102,7 +102,7 @@ def create_deprecated_class(
msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
new=_clspath(new_class, new_class_path))
warnings.warn(msg, warn_category, stacklevel=2)
return super(DeprecatedClass, cls).__call__(*args, **kwargs)
return super().__call__(*args, **kwargs)
deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})

2
scrapy/utils/log.py
View File

@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler):
"""Record log levels count into a crawler stats"""
def __init__(self, crawler, *args, **kwargs):
super(LogCounterHandler, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.crawler = crawler
def emit(self, record):

2
scrapy/utils/serialize.py
View File

@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
elif isinstance(o, Response):
return "<%s %s %s>" % (type(o).__name__, o.status, o.url)
else:
return super(ScrapyJSONEncoder, self).default(o)
return super().default(o)
class ScrapyJSONDecoder(json.JSONDecoder):

4
scrapy/utils/testsite.py
View File

@ -7,12 +7,12 @@ class SiteTest:
def setUp(self):
from twisted.internet import reactor
super(SiteTest, self).setUp()
super().setUp()
self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
self.baseurl = "http://localhost:%d/" % self.site.getHost().port
def tearDown(self):
super(SiteTest, self).tearDown()
super().tearDown()
self.site.stopListening()
def url(self, path):

20
tests/spiders.py
View File

@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue
class MockServerSpider(Spider):
def __init__(self, mockserver=None, *args, **kwargs):
super(MockServerSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.mockserver = mockserver
@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider):
name = 'meta'
def __init__(self, *args, **kwargs):
super(MetaSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.meta = {}
def closed(self, reason):
@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider):
link_extractor = LinkExtractor()
def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs):
super(FollowAllSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.urls_visited = []
self.times = []
qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency}
@ -60,7 +60,7 @@ class DelaySpider(MetaSpider):
name = 'delay'
def __init__(self, n=1, b=0, *args, **kwargs):
super(DelaySpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.n = n
self.b = b
self.t1 = self.t2 = self.t2_err = 0
@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider):
name = 'simple'
def __init__(self, url="http://localhost:8998", *args, **kwargs):
super(SimpleSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.start_urls = [url]
def parse(self, response):
@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider):
name = 'item'
def parse(self, response):
for request in super(ItemSpider, self).parse(response):
for request in super().parse(response):
yield request
yield Item()
yield {}
@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider):
raise self.exception_cls('Expected exception')
def parse(self, response):
for request in super(ErrorSpider, self).parse(response):
for request in super().parse(response):
yield request
self.raise_exception()
@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
fail_yielding = False
def __init__(self, *a, **kw):
super(BrokenStartRequestsSpider, self).__init__(*a, **kw)
super().__init__(*a, **kw)
self.seedsseen = []
def start_requests(self):
@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
def parse(self, response):
self.seedsseen.append(response.meta.get('seed'))
for req in super(BrokenStartRequestsSpider, self).parse(response):
for req in super().parse(response):
yield req
@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider):
yield Request(url, dont_filter=self.dont_filter)
def __init__(self, url="http://localhost:8998", *args, **kwargs):
super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.visited = 0
def parse(self, response):

2
tests/test_command_parse.py
View File

@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
command = 'parse'
def setUp(self):
super(ParseCommandTest, self).setUp()
super().setUp()
self.spider_name = 'parse_spider'
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
with open(fname, 'w') as f:

4
tests/test_commands.py
View File

@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None):
class StartprojectTemplatesTest(ProjectTest):
def setUp(self):
super(StartprojectTemplatesTest, self).setUp()
super().setUp()
self.tmpl = join(self.temp_path, 'templates')
self.tmpl_proj = join(self.tmpl, 'project')
@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest):
class CommandTest(ProjectTest):
def setUp(self):
super(CommandTest, self).setUp()
super().setUp()
self.call('startproject', self.project_name)
self.cwd = join(self.temp_path, self.project_name)
self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name

2
tests/test_contracts.py
View File

@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase):
name = 'test_same_url'
def __init__(self, *args, **kwargs):
super(TestSameUrlSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.visited = 0
def start_requests(s):

4
tests/test_downloader_handlers.py
View File

@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase):
"""Connect to HTTPS hosts with IP while certificate uses domain names IDs."""
def setUp(self):
super(Https11InvalidDNSId, self).setUp()
super().setUp()
self.host = '127.0.0.1'
@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", '
'subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
)
super(Https11InvalidDNSPattern, self).setUp()
super().setUp()
class Https11CustomCiphers(unittest.TestCase):

4
tests/test_downloadermiddleware_httpcache.py
View File

@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest):
def _get_settings(self, **new_settings):
new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module)
return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings)
return super()._get_settings(**new_settings)
def test_custom_dbm_module_loaded(self):
# make sure our dbm module has been loaded
@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
def _get_settings(self, **new_settings):
new_settings.setdefault('HTTPCACHE_GZIP', True)
return super(FilesystemStorageTest, self)._get_settings(**new_settings)
return super()._get_settings(**new_settings)
class DummyPolicyTest(_BaseTest):

4
tests/test_downloadermiddleware_robotstxt.py
View File

@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
skip = "Rerp parser is not installed"
def setUp(self):
super(RobotsTxtMiddlewareWithRerpTest, self).setUp()
super().setUp()
self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser')
@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest):
skip = "Reppy parser is not installed"
def setUp(self):
super(RobotsTxtMiddlewareWithReppyTest, self).setUp()
super().setUp()
self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser')

2
tests/test_exporters.py
View File

@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase):
if name == 'age':
return str(int(value) + 1)
else:
return super(CustomItemExporter, self).serialize_field(field, name, value)
return super().serialize_field(field, name, value)
i = self.item_class(name='John', age='22')
a = ItemAdapter(i)

4
tests/test_http_request.py
View File

@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest):
def setUp(self):
warnings.simplefilter("always")
super(JsonRequestTest, self).setUp()
super().setUp()
def test_data(self):
r1 = self.request_class(url="http://www.example.com/")
@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest):
def tearDown(self):
warnings.resetwarnings()
super(JsonRequestTest, self).tearDown()
super().tearDown()
if __name__ == "__main__":

2
tests/test_http_response.py
View File

@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest):
response_class = TextResponse
def test_replace(self):
super(TextResponseTest, self).test_replace()
super().test_replace()
r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")
r2 = r1.replace(url="http://www.example.com/other")
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")

2
tests/test_item.py
View File

@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase):
# requirement. When not done properly raises an error:
# TypeError: __class__ set to <class '__main__.MyItem'>
# defining 'MyItem' as <class '__main__.MyItem'>
super(MyItem, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
class DictItemTest(unittest.TestCase):

2
tests/test_linkextractors.py
View File

@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
])
def test_restrict_xpaths_with_html_entities(self):
super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
super().test_restrict_xpaths_with_html_entities()
def test_filteringlinkextractor_deprecation_warning(self):
"""Make sure the FilteringLinkExtractor deprecation warning is not

2
tests/test_loader.py
View File

@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase):
temp = Field()
def __init__(self, *args, **kwargs):
super(TempItem, self).__init__(self, *args, **kwargs)
super().__init__(self, *args, **kwargs)
self.setdefault('temp', 0.3)
class TempLoader(ItemLoader):

2
tests/test_loader_deprecated.py
View File

@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase):
class TempDict(dict):
def __init__(self, *args, **kwargs):
super(TempDict, self).__init__(self, *args, **kwargs)
super().__init__(self, *args, **kwargs)
self.setdefault('temp', 0.3)
class TempLoader(ItemLoader):

2
tests/test_logformatter.py
View File

@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase):
class LogFormatterSubclass(LogFormatter):
def crawled(self, request, response, spider):
kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
kwargs = super().crawled(request, response, spider)
CRAWLEDMSG = (
"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
)

2
tests/test_middleware.py
View File

@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager):
return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
def _add_middleware(self, mw):
super(TestMiddlewareManager, self)._add_middleware(mw)
super()._add_middleware(mw)
if hasattr(mw, 'process'):
self.methods['process'].append(mw.process)

12
tests/test_pipeline_media.py
View File

@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
class MockedMediaPipeline(MediaPipeline):
def __init__(self, *args, **kwargs):
super(MockedMediaPipeline, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self._mockcalled = []
def download(self, request, info):
self._mockcalled.append('download')
return super(MockedMediaPipeline, self).download(request, info)
return super().download(request, info)
def media_to_download(self, request, info):
self._mockcalled.append('media_to_download')
if 'result' in request.meta:
return request.meta.get('result')
return super(MockedMediaPipeline, self).media_to_download(request, info)
return super().media_to_download(request, info)
def get_media_requests(self, item, info):
self._mockcalled.append('get_media_requests')
@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline):
def media_downloaded(self, response, request, info):
self._mockcalled.append('media_downloaded')
return super(MockedMediaPipeline, self).media_downloaded(response, request, info)
return super().media_downloaded(response, request, info)
def media_failed(self, failure, request, info):
self._mockcalled.append('media_failed')
return super(MockedMediaPipeline, self).media_failed(failure, request, info)
return super().media_failed(failure, request, info)
def item_completed(self, results, item, info):
self._mockcalled.append('item_completed')
item = super(MockedMediaPipeline, self).item_completed(results, item, info)
item = super().item_completed(results, item, info)
item['results'] = results
return item

2
tests/test_request_left.py
View File

@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider):
name = 'signal_catcher'
def __init__(self, crawler, url, *args, **kwargs):
super(SignalCatcherSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
crawler.signals.connect(self.on_request_left,
signal=request_left_downloader)
self.caught_times = 0

8
tests/test_robotstxt_interface.py
View File

@ -117,7 +117,7 @@ class BaseRobotParserTest:
class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import PythonRobotParser
super(PythonRobotParserTest, self)._setUp(PythonRobotParser)
super()._setUp(PythonRobotParser)
def test_length_based_precedence(self):
raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.")
@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import ReppyRobotParser
super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser)
super()._setUp(ReppyRobotParser)
def test_order_based_precedence(self):
raise unittest.SkipTest("Reppy does not support order based directives precedence.")
@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import RerpRobotParser
super(RerpRobotParserTest, self)._setUp(RerpRobotParser)
super()._setUp(RerpRobotParser)
def test_length_based_precedence(self):
raise unittest.SkipTest("Rerp does not support length based directives precedence.")
@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase):
def setUp(self):
from scrapy.robotstxt import ProtegoRobotParser
super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser)
super()._setUp(ProtegoRobotParser)
def test_order_based_precedence(self):
raise unittest.SkipTest("Protego does not support order based directives precedence.")

4
tests/test_scheduler.py
View File

@ -53,7 +53,7 @@ class MockCrawler(Crawler):
JOBDIR=jobdir,
DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
)
super(MockCrawler, self).__init__(Spider, settings)
super().__init__(Spider, settings)
self.engine = MockEngine(downloader=MockDownloader())
@ -296,7 +296,7 @@ class StartUrlsSpider(Spider):
def __init__(self, start_urls):
self.start_urls = start_urls
super(StartUrlsSpider, self).__init__(name='StartUrlsSpider')
super().__init__(name='StartUrlsSpider')
def parse(self, response):
pass

2
tests/test_spidermiddleware_httperror.py
View File

@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider):
bypass_status_codes = set()
def __init__(self, *args, **kwargs):
super(_HttpErrorSpider, self).__init__(*args, **kwargs)
super().__init__(*args, **kwargs)
self.start_urls = [
self.mockserver.url("/status?n=200"),
self.mockserver.url("/status?n=404"),