From 336f19f5cc6edd0392c77f38b857d3e40bf565da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Hern=C3=A1ndez?= Date: Tue, 4 Aug 2020 20:42:01 +0200 Subject: [PATCH] Change super syntax (#4707) --- extras/qpsclient.py | 2 +- scrapy/commands/view.py | 2 +- scrapy/contracts/default.py | 2 +- scrapy/core/downloader/contextfactory.py | 4 ++-- scrapy/core/downloader/handlers/http11.py | 10 +++++----- scrapy/core/downloader/tls.py | 2 +- scrapy/core/spidermw.py | 2 +- scrapy/crawler.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/exceptions.py | 4 ++-- scrapy/exporters.py | 2 +- scrapy/http/headers.py | 8 ++++---- scrapy/http/request/form.py | 2 +- scrapy/http/request/json_request.py | 4 ++-- scrapy/http/request/rpc.py | 2 +- scrapy/http/response/text.py | 10 +++++----- scrapy/item.py | 10 +++++----- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/pipelines/files.py | 2 +- scrapy/pipelines/images.py | 3 +-- scrapy/resolver.py | 10 +++++----- scrapy/selector/unified.py | 2 +- scrapy/settings/__init__.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spiders/crawl.py | 4 ++-- scrapy/spiders/init.py | 2 +- scrapy/spiders/sitemap.py | 2 +- scrapy/squeues.py | 12 ++++++------ scrapy/statscollectors.py | 2 +- scrapy/utils/datatypes.py | 14 +++++++------- scrapy/utils/deprecate.py | 8 ++++---- scrapy/utils/log.py | 2 +- scrapy/utils/serialize.py | 2 +- scrapy/utils/testsite.py | 4 ++-- tests/spiders.py | 20 ++++++++++---------- tests/test_command_parse.py | 2 +- tests/test_commands.py | 4 ++-- tests/test_contracts.py | 2 +- tests/test_downloader_handlers.py | 4 ++-- tests/test_downloadermiddleware_httpcache.py | 4 ++-- tests/test_downloadermiddleware_robotstxt.py | 4 ++-- tests/test_exporters.py | 2 +- tests/test_http_request.py | 4 ++-- tests/test_http_response.py | 2 +- tests/test_item.py | 2 +- tests/test_linkextractors.py | 2 +- tests/test_loader.py | 2 +- tests/test_loader_deprecated.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 2 +- tests/test_pipeline_media.py | 12 ++++++------ tests/test_request_left.py | 2 +- tests/test_robotstxt_interface.py | 8 ++++---- tests/test_scheduler.py | 4 ++-- tests/test_spidermiddleware_httperror.py | 2 +- 56 files changed, 117 insertions(+), 118 deletions(-) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index 7554f7eec..fe1f96cbb 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -27,7 +27,7 @@ class QPSSpider(Spider): slots = 1 def __init__(self, *a, **kw): - super(QPSSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) if self.qps is not None: self.qps = float(self.qps) self.download_delay = 1 / self.qps diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 908bee966..c8f873334 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -11,7 +11,7 @@ class Command(fetch.Command): return "Fetch a URL using the Scrapy downloader and show its contents in a browser" def add_options(self, parser): - super(Command, self).add_options(parser) + super().add_options(parser) parser.remove_option("--headers") def _print_response(self, response, opts): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 34f0d36d4..cfdcc7c25 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -56,7 +56,7 @@ class ReturnsContract(Contract): } def __init__(self, *args, **kwargs): - super(ReturnsContract, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if len(self.args) not in [1, 2, 3]: raise ValueError( diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 452242d47..8a7d656a1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs): - super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._ssl_method = method self.tls_verbose_logging = tls_verbose_logging if tls_ciphers: @@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) # # * getattr() for `_ssl_method` attribute for context factories - # not calling super(..., self).__init__ + # not calling super().__init__ return CertificateOptions( verify=False, method=getattr(self, 'method', getattr(self, '_ssl_method', None)), diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 22c9ac520..fb04d1fb7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf - super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) + super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() self._tunneledHost = host self._tunneledPort = port @@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def connect(self, protocolFactory): self._protocolFactory = protocolFactory - connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) + connectDeferred = super().connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) connectDeferred.addErrback(self.connectFailed) return self._tunnelReadyDeferred @@ -215,7 +215,7 @@ class TunnelingAgent(Agent): def __init__(self, reactor, proxyConf, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None): - super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) + super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf = proxyConf self._contextFactory = contextFactory @@ -235,7 +235,7 @@ class TunnelingAgent(Agent): # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy key = key + self._proxyConf - return super(TunnelingAgent, self)._requestWithEndpoint( + return super()._requestWithEndpoint( key=key, endpoint=endpoint, method=method, @@ -249,7 +249,7 @@ class TunnelingAgent(Agent): class ScrapyProxyAgent(Agent): def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None): - super(ScrapyProxyAgent, self).__init__( + super().__init__( reactor=reactor, connectTimeout=connectTimeout, bindAddress=bindAddress, diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index e43a3c83e..d9f3750d5 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions): """ def __init__(self, hostname, ctx, verbose_logging=False): - super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + super().__init__(hostname, ctx) self.verbose_logging = verbose_logging def _identityVerifyingInfoCallback(self, connection, where, ret): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 35264a92b..5a99b96be 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) def _add_middleware(self, mw): - super(SpiderMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6f43771e2..48f19424c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner): """ def __init__(self, settings=None, install_root_handler=True): - super(CrawlerProcess, self).__init__(settings) + super().__init__(settings) install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 366d60dcb..4053fecc5 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -92,7 +92,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): enabled_setting = 'METAREFRESH_ENABLED' def __init__(self, settings): - super(MetaRefreshMiddleware, self).__init__(settings) + super().__init__(settings) self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') self._maxdelay = settings.getint('METAREFRESH_MAXDELAY') diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 45f152321..0c410f035 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -37,7 +37,7 @@ class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" def __init__(self, reason='cancelled'): - super(CloseSpider, self).__init__() + super().__init__() self.reason = reason @@ -74,7 +74,7 @@ class UsageError(Exception): def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) - super(UsageError, self).__init__(*a, **kw) + super().__init__(*a, **kw) class ScrapyDeprecationWarning(Warning): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0aba1c904..95518b3ac 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -301,7 +301,7 @@ class PythonItemExporter(BaseItemExporter): def _configure(self, options, dont_fail=False): self.binary = options.pop('binary', True) - super(PythonItemExporter, self)._configure(options, dont_fail) + super()._configure(options, dont_fail) if self.binary: warnings.warn( "PythonItemExporter will drop support for binary export in the future", diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index dcaaeddfa..6bf9e5346 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -8,7 +8,7 @@ class Headers(CaselessDict): def __init__(self, seq=None, encoding='utf-8'): self.encoding = encoding - super(Headers, self).__init__(seq) + super().__init__(seq) def normkey(self, key): """Normalize key to bytes""" @@ -37,19 +37,19 @@ class Headers(CaselessDict): def __getitem__(self, key): try: - return super(Headers, self).__getitem__(key)[-1] + return super().__getitem__(key)[-1] except IndexError: return None def get(self, key, def_val=None): try: - return super(Headers, self).get(key, def_val)[-1] + return super().get(key, def_val)[-1] except IndexError: return None def getlist(self, key, def_val=None): try: - return super(Headers, self).__getitem__(key) + return super().__getitem__(key) except KeyError: if def_val is not None: return self.normvalue(def_val) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a260798ac..59af81321 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -24,7 +24,7 @@ class FormRequest(Request): if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' - super(FormRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) if formdata: items = formdata.items() if isinstance(formdata, dict) else formdata diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index f08b25280..eae3f9f6b 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -32,7 +32,7 @@ class JsonRequest(Request): if 'method' not in kwargs: kwargs['method'] = 'POST' - super(JsonRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'application/json') self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @@ -47,7 +47,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - return super(JsonRequest, self).replace(*args, **kwargs) + return super().replace(*args, **kwargs) def _dumps(self, data): """Convert to JSON """ diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 811d3ad6b..c70912e49 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -31,5 +31,5 @@ class XmlRpcRequest(Request): if encoding is not None: kwargs['encoding'] = encoding - super(XmlRpcRequest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'text/xml') diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0f300c8da..a7bb34d48 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -35,13 +35,13 @@ class TextResponse(Response): self._cached_benc = None self._cached_ubody = None self._cached_selector = None - super(TextResponse, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def _set_url(self, url): if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: - super(TextResponse, self)._set_url(url) + super()._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection @@ -51,7 +51,7 @@ class TextResponse(Response): type(self).__name__) self._body = body.encode(self._encoding) else: - super(TextResponse, self)._set_body(body) + super()._set_body(body) def replace(self, *args, **kwargs): kwargs.setdefault('encoding', self.encoding) @@ -166,7 +166,7 @@ class TextResponse(Response): elif isinstance(url, parsel.SelectorList): raise ValueError("SelectorList is not supported") encoding = self.encoding if encoding is None else encoding - return super(TextResponse, self).follow( + return super().follow( url=url, callback=callback, method=method, @@ -226,7 +226,7 @@ class TextResponse(Response): for sel in selectors: with suppress(_InvalidSelector): urls.append(_url_from_selector(sel)) - return super(TextResponse, self).follow_all( + return super().follow_all( urls=urls, callback=callback, method=method, diff --git a/scrapy/item.py b/scrapy/item.py index 4ab83d1a0..c262a153c 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta): if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)): warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(BaseItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) class Field(dict): @@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta): def __new__(mcs, class_name, bases, attrs): classcell = attrs.pop('__classcell__', None) new_bases = tuple(base._class for base in bases if hasattr(base, '_class')) - _class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs) + _class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs) fields = getattr(_class, 'fields', {}) new_attrs = {} @@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta): new_attrs['_class'] = _class if classcell is not None: new_attrs['__classcell__'] = classcell - return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs) + return super().__new__(mcs, class_name, bases, new_attrs) class DictItem(MutableMapping, BaseItem): @@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem): if issubclass(cls, DictItem) and not issubclass(cls, Item): warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead', ScrapyDeprecationWarning, stacklevel=2) - return super(DictItem, cls).__new__(cls, *args, **kwargs) + return super().__new__(cls, *args, **kwargs) def __init__(self, *args, **kwargs): self._values = {} @@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem): def __setattr__(self, name, value): if not name.startswith('_'): raise AttributeError("Use item[%r] = %r to set field value" % (name, value)) - super(DictItem, self).__setattr__(name, value) + super().__setattr__(name, value) def __len__(self): return len(self._values) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index 984a5c4e1..08a6ca1e8 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -65,7 +65,7 @@ class FilteringLinkExtractor: warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, ' 'please use scrapy.linkextractors.LinkExtractor instead', ScrapyDeprecationWarning, stacklevel=2) - return super(FilteringLinkExtractor, cls).__new__(cls) + return super().__new__(cls) def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text): diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 8b9f961ee..e941c4321 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor): strip=strip, canonicalized=canonicalize ) - super(LxmlLinkExtractor, self).__init__( + super().__init__( link_extractor=lx, allow=allow, deny=deny, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 487382a38..6bc5d46eb 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline): resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD ) - super(FilesPipeline, self).__init__(download_func=download_func, settings=settings) + super().__init__(download_func=download_func, settings=settings) @classmethod def from_settings(cls, settings): diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 46f2bfb58..e2dd70215 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline): DEFAULT_IMAGES_RESULT_FIELD = 'images' def __init__(self, store_uri, download_func=None, settings=None): - super(ImagesPipeline, self).__init__(store_uri, settings=settings, - download_func=download_func) + super().__init__(store_uri, settings=settings, download_func=download_func) if isinstance(settings, dict) or settings is None: settings = Settings(settings) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..f191deac6 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver): """ def __init__(self, reactor, cache_size, timeout): - super(CachingThreadedResolver, self).__init__(reactor) + super().__init__(reactor) dnscache.limit = cache_size self.timeout = timeout @@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver): # so the input argument above is simply overridden # to enforce Scrapy's DNS_TIMEOUT setting's value timeout = (self.timeout,) - d = super(CachingThreadedResolver, self).getHostByName(name, timeout) + d = super().getHostByName(name, timeout) if dnscache.limit: d.addCallback(self._cache_result, name) return d @@ -80,16 +80,16 @@ class CachingHostnameResolver: class CachingResolutionReceiver(resolutionReceiver): def resolutionBegan(self, resolution): - super(CachingResolutionReceiver, self).resolutionBegan(resolution) + super().resolutionBegan(resolution) self.resolution = resolution self.resolved = False def addressResolved(self, address): - super(CachingResolutionReceiver, self).addressResolved(address) + super().addressResolved(address) self.resolved = True def resolutionComplete(self): - super(CachingResolutionReceiver, self).resolutionComplete() + super().resolutionComplete() if self.resolved: dnscache[hostName] = self.resolution diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 85a9bb526..f12c61081 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref): kwargs.setdefault('base_url', response.url) self.response = response - super(Selector, self).__init__(text=text, type=st, root=root, **kwargs) + super().__init__(text=text, type=st, root=root, **kwargs) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index ff8317cd1..b8ae32d7c 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -439,7 +439,7 @@ class Settings(BaseSettings): # Do not pass kwarg values here. We don't want to promote user-defined # dicts, and we want to update, not replace, default dicts with the # values given by the user - super(Settings, self).__init__() + super().__init__() self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 375042340..db9d0f2ae 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -15,7 +15,7 @@ class HttpError(IgnoreRequest): def __init__(self, response, *args, **kwargs): self.response = response - super(HttpError, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class HttpErrorMiddleware: diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index cb7260892..c9fbce08d 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -75,7 +75,7 @@ class CrawlSpider(Spider): rules = () def __init__(self, *a, **kw): - super(CrawlSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._compile_rules() def _parse(self, response, **kwargs): @@ -145,6 +145,6 @@ class CrawlSpider(Spider): @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs) + spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) return spider diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index fd41133ea..fe8c94e78 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -6,7 +6,7 @@ class InitSpider(Spider): """Base Spider with initialization facilities""" def start_requests(self): - self._postinit_reqs = super(InitSpider, self).start_requests() + self._postinit_reqs = super().start_requests() return iterate_spider_output(self.init_request()) def initialized(self, response=None): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c5360bfa7..1f72e76b7 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -18,7 +18,7 @@ class SitemapSpider(Spider): sitemap_alternate_links = False def __init__(self, *a, **kw): - super(SitemapSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: if isinstance(c, str): diff --git a/scrapy/squeues.py b/scrapy/squeues.py index c7ad4d53d..77ffda6f7 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -20,7 +20,7 @@ def _with_mkdir(queue_class): if not os.path.exists(dirname): os.makedirs(dirname, exist_ok=True) - super(DirectoriesCreated, self).__init__(path, *args, **kwargs) + super().__init__(path, *args, **kwargs) return DirectoriesCreated @@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def push(self, obj): s = serialize(obj) - super(SerializableQueue, self).push(s) + super().push(s) def pop(self): - s = super(SerializableQueue, self).pop() + s = super().pop() if s: return deserialize(s) @@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class): def __init__(self, crawler, key): self.spider = crawler.spider - super(ScrapyRequestQueue, self).__init__(key) + super().__init__(key) @classmethod def from_crawler(cls, crawler, key, *args, **kwargs): @@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class): def push(self, request): request = request_to_dict(request, self.spider) - return super(ScrapyRequestQueue, self).push(request) + return super().push(request) def pop(self): - request = super(ScrapyRequestQueue, self).pop() + request = super().pop() if not request: return None diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index 579c60180..ba7d1a6bf 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -54,7 +54,7 @@ class StatsCollector: class MemoryStatsCollector(StatsCollector): def __init__(self, crawler): - super(MemoryStatsCollector, self).__init__(crawler) + super().__init__(crawler) self.spider_stats = {} def _persist_stats(self, stats, spider): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 2a92d0588..e31284a7f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -15,7 +15,7 @@ class CaselessDict(dict): __slots__ = () def __init__(self, seq=None): - super(CaselessDict, self).__init__() + super().__init__() if seq: self.update(seq) @@ -53,7 +53,7 @@ class CaselessDict(dict): def update(self, seq): seq = seq.items() if isinstance(seq, Mapping) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) - super(CaselessDict, self).update(iseq) + super().update(iseq) @classmethod def fromkeys(cls, keys, value=None): @@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict): """ def __init__(self, limit=None): - super(LocalCache, self).__init__() + super().__init__() self.limit = limit def __setitem__(self, key, value): if self.limit: while len(self) >= self.limit: self.popitem(last=False) - super(LocalCache, self).__setitem__(key, value) + super().__setitem__(key, value) class LocalWeakReferencedCache(weakref.WeakKeyDictionary): @@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): """ def __init__(self, limit=None): - super(LocalWeakReferencedCache, self).__init__() + super().__init__() self.data = LocalCache(limit=limit) def __setitem__(self, key, value): try: - super(LocalWeakReferencedCache, self).__setitem__(key, value) + super().__setitem__(key, value) except TypeError: pass # key is not weak-referenceable, skip caching def __getitem__(self, key): try: - return super(LocalWeakReferencedCache, self).__getitem__(key) + return super().__getitem__(key) except (TypeError, KeyError): return None # key is either not weak-referenceable or not cached diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3dbea5fee..3c8e3c8b5 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -57,7 +57,7 @@ def create_deprecated_class( warned_on_subclass = False def __new__(metacls, name, bases, clsdict_): - cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_) + cls = super().__new__(metacls, name, bases, clsdict_) if metacls.deprecated_class is None: metacls.deprecated_class = cls return cls @@ -73,7 +73,7 @@ def create_deprecated_class( if warn_once: msg += ' (warning only on first subclass, there may be others)' warnings.warn(msg, warn_category, stacklevel=2) - super(DeprecatedClass, cls).__init__(name, bases, clsdict_) + super().__init__(name, bases, clsdict_) # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks @@ -88,7 +88,7 @@ def create_deprecated_class( # is the deprecated class itself - subclasses of the # deprecated class should not use custom `__subclasscheck__` # method. - return super(DeprecatedClass, cls).__subclasscheck__(sub) + return super().__subclasscheck__(sub) if not inspect.isclass(sub): raise TypeError("issubclass() arg 1 must be a class") @@ -102,7 +102,7 @@ def create_deprecated_class( msg = instance_warn_message.format(cls=_clspath(cls, old_class_path), new=_clspath(new_class, new_class_path)) warnings.warn(msg, warn_category, stacklevel=2) - return super(DeprecatedClass, cls).__call__(*args, **kwargs) + return super().__call__(*args, **kwargs) deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {}) diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 51d276097..1d6a2c39d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler): """Record log levels count into a crawler stats""" def __init__(self, crawler, *args, **kwargs): - super(LogCounterHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.crawler = crawler def emit(self, record): diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index dc9604578..cc3263602 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder): elif isinstance(o, Response): return "<%s %s %s>" % (type(o).__name__, o.status, o.url) else: - return super(ScrapyJSONEncoder, self).default(o) + return super().default(o) class ScrapyJSONDecoder(json.JSONDecoder): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 66930ad2c..397e54703 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -7,12 +7,12 @@ class SiteTest: def setUp(self): from twisted.internet import reactor - super(SiteTest, self).setUp() + super().setUp() self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1") self.baseurl = "http://localhost:%d/" % self.site.getHost().port def tearDown(self): - super(SiteTest, self).tearDown() + super().tearDown() self.site.stopListening() def url(self, path): diff --git a/tests/spiders.py b/tests/spiders.py index 3eb681819..63bd726fb 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -19,7 +19,7 @@ from scrapy.utils.test import get_from_asyncio_queue class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): - super(MockServerSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.mockserver = mockserver @@ -28,7 +28,7 @@ class MetaSpider(MockServerSpider): name = 'meta' def __init__(self, *args, **kwargs): - super(MetaSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.meta = {} def closed(self, reason): @@ -41,7 +41,7 @@ class FollowAllSpider(MetaSpider): link_extractor = LinkExtractor() def __init__(self, total=10, show=20, order="rand", maxlatency=0.0, *args, **kwargs): - super(FollowAllSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.urls_visited = [] self.times = [] qargs = {'total': total, 'show': show, 'order': order, 'maxlatency': maxlatency} @@ -60,7 +60,7 @@ class DelaySpider(MetaSpider): name = 'delay' def __init__(self, n=1, b=0, *args, **kwargs): - super(DelaySpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.n = n self.b = b self.t1 = self.t2 = self.t2_err = 0 @@ -82,7 +82,7 @@ class SimpleSpider(MetaSpider): name = 'simple' def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(SimpleSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [url] def parse(self, response): @@ -153,7 +153,7 @@ class ItemSpider(FollowAllSpider): name = 'item' def parse(self, response): - for request in super(ItemSpider, self).parse(response): + for request in super().parse(response): yield request yield Item() yield {} @@ -172,7 +172,7 @@ class ErrorSpider(FollowAllSpider): raise self.exception_cls('Expected exception') def parse(self, response): - for request in super(ErrorSpider, self).parse(response): + for request in super().parse(response): yield request self.raise_exception() @@ -183,7 +183,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): fail_yielding = False def __init__(self, *a, **kw): - super(BrokenStartRequestsSpider, self).__init__(*a, **kw) + super().__init__(*a, **kw) self.seedsseen = [] def start_requests(self): @@ -201,7 +201,7 @@ class BrokenStartRequestsSpider(FollowAllSpider): def parse(self, response): self.seedsseen.append(response.meta.get('seed')) - for req in super(BrokenStartRequestsSpider, self).parse(response): + for req in super().parse(response): yield req @@ -243,7 +243,7 @@ class DuplicateStartRequestsSpider(MockServerSpider): yield Request(url, dont_filter=self.dont_filter) def __init__(self, url="http://localhost:8998", *args, **kwargs): - super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def parse(self, response): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5754a5478..e115f420f 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -17,7 +17,7 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' def setUp(self): - super(ParseCommandTest, self).setUp() + super().setUp() self.spider_name = 'parse_spider' fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py')) with open(fname, 'w') as f: diff --git a/tests/test_commands.py b/tests/test_commands.py index 42091ab00..8938156fc 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -151,7 +151,7 @@ def get_permissions_dict(path, renamings=None, ignore=None): class StartprojectTemplatesTest(ProjectTest): def setUp(self): - super(StartprojectTemplatesTest, self).setUp() + super().setUp() self.tmpl = join(self.temp_path, 'templates') self.tmpl_proj = join(self.tmpl, 'project') @@ -315,7 +315,7 @@ class StartprojectTemplatesTest(ProjectTest): class CommandTest(ProjectTest): def setUp(self): - super(CommandTest, self).setUp() + super().setUp() self.call('startproject', self.project_name) self.cwd = join(self.temp_path, self.project_name) self.env['SCRAPY_SETTINGS_MODULE'] = '%s.settings' % self.project_name diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 99120b128..2e7e3ccc4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -378,7 +378,7 @@ class ContractsManagerTest(unittest.TestCase): name = 'test_same_url' def __init__(self, *args, **kwargs): - super(TestSameUrlSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.visited = 0 def start_requests(s): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 57d4cdd6b..13063d106 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -530,7 +530,7 @@ class Https11InvalidDNSId(Https11TestCase): """Connect to HTTPS hosts with IP while certificate uses domain names IDs.""" def setUp(self): - super(Https11InvalidDNSId, self).setUp() + super().setUp() self.host = '127.0.0.1' @@ -549,7 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase): 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", ' 'subject "/C=IE/O=Scrapy/CN=127.0.0.1"' ) - super(Https11InvalidDNSPattern, self).setUp() + super().setUp() class Https11CustomCiphers(unittest.TestCase): diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9b77c97a8..299fb0eb8 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -134,7 +134,7 @@ class DbmStorageWithCustomDbmModuleTest(DbmStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_DBM_MODULE', self.dbm_module) - return super(DbmStorageWithCustomDbmModuleTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) def test_custom_dbm_module_loaded(self): # make sure our dbm module has been loaded @@ -151,7 +151,7 @@ class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): new_settings.setdefault('HTTPCACHE_GZIP', True) - return super(FilesystemStorageTest, self)._get_settings(**new_settings) + return super()._get_settings(**new_settings) class DummyPolicyTest(_BaseTest): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index f9936baba..858138f81 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -189,7 +189,7 @@ class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): skip = "Rerp parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithRerpTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.RerpRobotParser') @@ -198,5 +198,5 @@ class RobotsTxtMiddlewareWithReppyTest(RobotsTxtMiddlewareTest): skip = "Reppy parser is not installed" def setUp(self): - super(RobotsTxtMiddlewareWithReppyTest, self).setUp() + super().setUp() self.crawler.settings.set('ROBOTSTXT_PARSER', 'scrapy.robotstxt.ReppyRobotParser') diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 660c99ce1..6c25a0064 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -593,7 +593,7 @@ class CustomExporterItemTest(unittest.TestCase): if name == 'age': return str(int(value) + 1) else: - return super(CustomItemExporter, self).serialize_field(field, name, value) + return super().serialize_field(field, name, value) i = self.item_class(name='John', age='22') a = ItemAdapter(i) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index f5cf4e798..0a303dbe2 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1265,7 +1265,7 @@ class JsonRequestTest(RequestTest): def setUp(self): warnings.simplefilter("always") - super(JsonRequestTest, self).setUp() + super().setUp() def test_data(self): r1 = self.request_class(url="http://www.example.com/") @@ -1419,7 +1419,7 @@ class JsonRequestTest(RequestTest): def tearDown(self): warnings.resetwarnings() - super(JsonRequestTest, self).tearDown() + super().tearDown() if __name__ == "__main__": diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 56d017de6..f831ef5dc 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -305,7 +305,7 @@ class TextResponseTest(BaseResponseTest): response_class = TextResponse def test_replace(self): - super(TextResponseTest, self).test_replace() + super().test_replace() r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852") r2 = r1.replace(url="http://www.example.com/other") r3 = r1.replace(url="http://www.example.com/other", encoding="latin1") diff --git a/tests/test_item.py b/tests/test_item.py index 0ce78f8c0..66fa761f0 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -312,7 +312,7 @@ class ItemMetaClassCellRegression(unittest.TestCase): # requirement. When not done properly raises an error: # TypeError: __class__ set to # defining 'MyItem' as - super(MyItem, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) class DictItemTest(unittest.TestCase): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index a0bafa5e5..6f133d77a 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -516,7 +516,7 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ]) def test_restrict_xpaths_with_html_entities(self): - super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities() + super().test_restrict_xpaths_with_html_entities() def test_filteringlinkextractor_deprecation_warning(self): """Make sure the FilteringLinkExtractor deprecation warning is not diff --git a/tests/test_loader.py b/tests/test_loader.py index 2ed6f365f..b0bc82f4e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -250,7 +250,7 @@ class TestOutputProcessorItem(unittest.TestCase): temp = Field() def __init__(self, *args, **kwargs): - super(TempItem, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index eb14de14f..624dd9ab8 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -579,7 +579,7 @@ class TestOutputProcessorDict(unittest.TestCase): class TempDict(dict): def __init__(self, *args, **kwargs): - super(TempDict, self).__init__(self, *args, **kwargs) + super().__init__(self, *args, **kwargs) self.setdefault('temp', 0.3) class TempLoader(ItemLoader): diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index b771e7d79..41ff3651d 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -118,7 +118,7 @@ class LogFormatterTestCase(unittest.TestCase): class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): - kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) + kwargs = super().crawled(request, response, spider) CRAWLEDMSG = ( "Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3364d2258..b2b75ef20 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -53,7 +53,7 @@ class TestMiddlewareManager(MiddlewareManager): return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']] def _add_middleware(self, mw): - super(TestMiddlewareManager, self)._add_middleware(mw) + super()._add_middleware(mw) if hasattr(mw, 'process'): self.methods['process'].append(mw.process) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 19ff00350..4f130c0c9 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -162,18 +162,18 @@ class BaseMediaPipelineTestCase(unittest.TestCase): class MockedMediaPipeline(MediaPipeline): def __init__(self, *args, **kwargs): - super(MockedMediaPipeline, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self._mockcalled = [] def download(self, request, info): self._mockcalled.append('download') - return super(MockedMediaPipeline, self).download(request, info) + return super().download(request, info) def media_to_download(self, request, info): self._mockcalled.append('media_to_download') if 'result' in request.meta: return request.meta.get('result') - return super(MockedMediaPipeline, self).media_to_download(request, info) + return super().media_to_download(request, info) def get_media_requests(self, item, info): self._mockcalled.append('get_media_requests') @@ -181,15 +181,15 @@ class MockedMediaPipeline(MediaPipeline): def media_downloaded(self, response, request, info): self._mockcalled.append('media_downloaded') - return super(MockedMediaPipeline, self).media_downloaded(response, request, info) + return super().media_downloaded(response, request, info) def media_failed(self, failure, request, info): self._mockcalled.append('media_failed') - return super(MockedMediaPipeline, self).media_failed(failure, request, info) + return super().media_failed(failure, request, info) def item_completed(self, results, item, info): self._mockcalled.append('item_completed') - item = super(MockedMediaPipeline, self).item_completed(results, item, info) + item = super().item_completed(results, item, info) item['results'] = results return item diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 5cfef8e7d..373b2e49c 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -10,7 +10,7 @@ class SignalCatcherSpider(Spider): name = 'signal_catcher' def __init__(self, crawler, url, *args, **kwargs): - super(SignalCatcherSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 9d8c201dd..4b15d0fab 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -117,7 +117,7 @@ class BaseRobotParserTest: class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import PythonRobotParser - super(PythonRobotParserTest, self)._setUp(PythonRobotParser) + super()._setUp(PythonRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("RobotFileParser does not support length based directives precedence.") @@ -132,7 +132,7 @@ class ReppyRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ReppyRobotParser - super(ReppyRobotParserTest, self)._setUp(ReppyRobotParser) + super()._setUp(ReppyRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Reppy does not support order based directives precedence.") @@ -144,7 +144,7 @@ class RerpRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import RerpRobotParser - super(RerpRobotParserTest, self)._setUp(RerpRobotParser) + super()._setUp(RerpRobotParser) def test_length_based_precedence(self): raise unittest.SkipTest("Rerp does not support length based directives precedence.") @@ -156,7 +156,7 @@ class ProtegoRobotParserTest(BaseRobotParserTest, unittest.TestCase): def setUp(self): from scrapy.robotstxt import ProtegoRobotParser - super(ProtegoRobotParserTest, self)._setUp(ProtegoRobotParser) + super()._setUp(ProtegoRobotParser) def test_order_based_precedence(self): raise unittest.SkipTest("Protego does not support order based directives precedence.") diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 2b6cb0902..512a7460e 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -53,7 +53,7 @@ class MockCrawler(Crawler): JOBDIR=jobdir, DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter', ) - super(MockCrawler, self).__init__(Spider, settings) + super().__init__(Spider, settings) self.engine = MockEngine(downloader=MockDownloader()) @@ -296,7 +296,7 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls - super(StartUrlsSpider, self).__init__(name='StartUrlsSpider') + super().__init__(name='StartUrlsSpider') def parse(self, response): pass diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index e032b247c..e449cd706 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -19,7 +19,7 @@ class _HttpErrorSpider(MockServerSpider): bypass_status_codes = set() def __init__(self, *args, **kwargs): - super(_HttpErrorSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [ self.mockserver.url("/status?n=200"), self.mockserver.url("/status?n=404"),