diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index d411e2eed..e52682690 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -42,7 +42,7 @@ value of one of their fields:: from scrapy.exporters import XmlItemExporter - class PerYearXmlExportPipeline(object): + class PerYearXmlExportPipeline: """Distribute items across multiple XML files according to their 'year' field""" def open_spider(self, spider): diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index dc057f6b6..94fd2e36e 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -107,7 +107,7 @@ Here is the code of such extension:: logger = logging.getLogger(__name__) - class SpiderOpenCloseLogging(object): + class SpiderOpenCloseLogging: def __init__(self, item_count): self.item_count = item_count diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 98e2506e5..533f84630 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -81,7 +81,7 @@ contain a price:: from scrapy.exceptions import DropItem - class PricePipeline(object): + class PricePipeline: vat_factor = 1.15 @@ -103,7 +103,7 @@ format:: import json - class JsonWriterPipeline(object): + class JsonWriterPipeline: def open_spider(self, spider): self.file = open('items.jl', 'w') @@ -132,7 +132,7 @@ method and how to clean up the resources properly.:: import pymongo - class MongoPipeline(object): + class MongoPipeline: collection_name = 'scrapy_items' @@ -180,7 +180,7 @@ it saves the screenshot to a file and adds filename to the item. from urllib.parse import quote - class ScreenshotPipeline(object): + class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of every Scrapy item.""" @@ -219,7 +219,7 @@ returns multiples items with the same id:: from scrapy.exceptions import DropItem - class DuplicatesPipeline(object): + class DuplicatesPipeline: def __init__(self): self.ids_seen = set() diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index c0c83fc84..4ee447065 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. .. class:: object_ref - Inherit from this class (instead of object) if you want to track live + Inherit from this class if you want to track live instances with the ``trackref`` module. .. function:: print_live_refs(class_name, ignore=NoneType) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c01202a10..dc6843d75 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -124,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in extensions, middlewares and item pipelines:: - class MyExtension(object): + class MyExtension: def __init__(self, log_is_enabled=False): if log_is_enabled: print("log is enabled!") diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 3dd829ebe..af848b402 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -32,7 +32,7 @@ Common Stats Collector uses Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats` attribute. Here is an example of an extension that access stats:: - class ExtensionThatAccessStats(object): + class ExtensionThatAccessStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0b24193c2..a573a03d9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -9,7 +9,7 @@ from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError -class ScrapyCommand(object): +class ScrapyCommand: requires_project = False crawler_process = None diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 7bbe362e7..c9f3b38e0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -25,7 +25,7 @@ class Command(ScrapyCommand): self.crawler_process.start() -class _BenchServer(object): +class _BenchServer: def __enter__(self): from scrapy.utils.test import get_testenv diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 7b6591d86..41d4f25b2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec -class ContractsManager(object): +class ContractsManager: contracts = {} def __init__(self, contracts): @@ -107,7 +107,7 @@ class ContractsManager(object): request.errback = eb_wrapper -class Contract(object): +class Contract: """ Abstract class for contracts """ request_cls = None diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 644be121f..36aca4dae 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -13,7 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers -class Slot(object): +class Slot: """Downloader slot""" def __init__(self, concurrency, delay, randomize_delay): @@ -66,7 +66,7 @@ def _get_concurrency_delay(concurrency, spider, settings): return concurrency, delay -class Downloader(object): +class Downloader: DOWNLOAD_SLOT = 'download_slot' diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 04a8d617a..c970909d7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent): ) -class ScrapyAgent(object): +class ScrapyAgent: _Agent = Agent _ProxyAgent = ScrapyProxyAgent @@ -432,7 +432,7 @@ class ScrapyAgent(object): @implementer(IBodyProducer) -class _RequestBodyProducer(object): +class _RequestBodyProducer: def __init__(self, body): self.body = body diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6ab8cde6b..74f03344e 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -21,7 +21,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info logger = logging.getLogger(__name__) -class Slot(object): +class Slot: def __init__(self, start_requests, close_if_idle, nextcall, scheduler): self.closing = False @@ -53,7 +53,7 @@ class Slot(object): self.closing.callback(None) -class ExecutionEngine(object): +class ExecutionEngine: def __init__(self, crawler, spider_closed_callback): self.crawler = crawler diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index c96b9b719..a18c26b17 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -14,7 +14,7 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning logger = logging.getLogger(__name__) -class Scheduler(object): +class Scheduler: """ Scrapy Scheduler. It allows to enqueue requests and then get a next request to download. Scheduler is also handling duplication diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 41f015017..3e4826216 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -21,7 +21,7 @@ from scrapy.core.spidermw import SpiderMiddlewareManager logger = logging.getLogger(__name__) -class Slot(object): +class Slot: """Scraper slot (one per running spider)""" MIN_RESPONSE_SIZE = 1024 @@ -62,7 +62,7 @@ class Slot(object): return self.active_size > self.max_active_size -class Scraper(object): +class Scraper: def __init__(self, crawler): self.slot = None diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 16b046e99..ad7a81e6b 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -11,7 +11,7 @@ from scrapy.http import HtmlResponse logger = logging.getLogger(__name__) -class AjaxCrawlMiddleware(object): +class AjaxCrawlMiddleware: """ Handle 'AJAX crawlable' pages marked as crawlable via meta tag. For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d8dabdf13..d57f04bc3 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -10,7 +10,7 @@ from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) -class CookiesMiddleware(object): +class CookiesMiddleware: """This middleware enables working with sites that need cookies""" def __init__(self, debug=False): diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index fcea38ef5..0fcf8fb8c 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) -class DecompressionMiddleware(object): +class DecompressionMiddleware: """ This middleware tries to recognise and extract the possibly compressed responses that may arrive. """ diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 93fe97673..f67961881 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from scrapy.utils.python import without_none_values -class DefaultHeadersMiddleware(object): +class DefaultHeadersMiddleware: def __init__(self, headers): self._headers = headers diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 18123cfce..d373a22df 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from scrapy import signals -class DownloadTimeoutMiddleware(object): +class DownloadTimeoutMiddleware: def __init__(self, timeout=180): self._timeout = timeout diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 7aa7a62bc..089bf0d85 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header from scrapy import signals -class HttpAuthMiddleware(object): +class HttpAuthMiddleware: """Set Basic HTTP Authorization header (http_user and http_pass spider class attributes)""" diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 4e06f8236..6db57bd8b 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.utils.misc import load_object -class HttpCacheMiddleware(object): +class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0010b2a8f..727c41466 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -15,7 +15,7 @@ except ImportError: pass -class HttpCompressionMiddleware(object): +class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" @classmethod diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 814ce78fe..da89d3e9b 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -7,7 +7,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes -class HttpProxyMiddleware(object): +class HttpProxyMiddleware: def __init__(self, auth_encoding='latin-1'): self.auth_encoding = auth_encoding diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 08cff8a55..09ee8377e 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) -class BaseRedirectMiddleware(object): +class BaseRedirectMiddleware: enabled_setting = 'REDIRECT_ENABLED' diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 7ab5b6e62..bbf5fca05 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -25,7 +25,7 @@ from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) -class RetryMiddleware(object): +class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to # decompress an empty response diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 251706c50..7f18b2bf2 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -16,7 +16,7 @@ from scrapy.utils.misc import load_object logger = logging.getLogger(__name__) -class RobotsTxtMiddleware(object): +class RobotsTxtMiddleware: DOWNLOAD_PRIORITY = 1000 def __init__(self, crawler): diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index ef0aafce0..46a2ad397 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -4,7 +4,7 @@ from scrapy.utils.response import response_httprepr from scrapy.utils.python import global_object_name -class DownloaderStats(object): +class DownloaderStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index d24750c69..3ee7bd129 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -3,7 +3,7 @@ from scrapy import signals -class UserAgentMiddleware(object): +class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" def __init__(self, user_agent='Scrapy'): diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index a36c8304f..d74c8ed36 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -5,7 +5,7 @@ from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint -class BaseDupeFilter(object): +class BaseDupeFilter: @classmethod def from_settings(cls, settings): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 2e20a7180..0cb6cef98 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -21,7 +21,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', 'JsonItemExporter', 'MarshalItemExporter'] -class BaseItemExporter(object): +class BaseItemExporter: def __init__(self, *, dont_fail=False, **kwargs): self._kwargs = kwargs diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 260b2e86e..e3f212bef 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -10,7 +10,7 @@ from scrapy import signals from scrapy.exceptions import NotConfigured -class CloseSpider(object): +class CloseSpider: def __init__(self, crawler): self.crawler = crawler diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 20adfbe4b..389cb65bc 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -6,7 +6,7 @@ from datetime import datetime from scrapy import signals -class CoreStats(object): +class CoreStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 625e13249..586399784 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -17,7 +17,7 @@ from scrapy.utils.trackref import format_live_refs logger = logging.getLogger(__name__) -class StackTraceDump(object): +class StackTraceDump: def __init__(self, crawler=None): self.crawler = crawler @@ -52,7 +52,7 @@ class StackTraceDump(object): return dumps -class Debugger(object): +class Debugger: def __init__(self): try: signal.signal(signal.SIGUSR2, self._enter_debugger) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 108b6d35c..998d2a5d1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -44,7 +44,7 @@ class IFeedStorage(Interface): @implementer(IFeedStorage) -class BlockingFeedStorage(object): +class BlockingFeedStorage: def open(self, spider): path = spider.crawler.settings['FEED_TEMPDIR'] @@ -61,7 +61,7 @@ class BlockingFeedStorage(object): @implementer(IFeedStorage) -class StdoutFeedStorage(object): +class StdoutFeedStorage: def __init__(self, uri, _stdout=None): if not _stdout: @@ -76,7 +76,7 @@ class StdoutFeedStorage(object): @implementer(IFeedStorage) -class FileFeedStorage(object): +class FileFeedStorage: def __init__(self, uri): self.path = file_uri_to_path(uri) @@ -179,7 +179,7 @@ class FTPFeedStorage(BlockingFeedStorage): ) -class _FeedSlot(object): +class _FeedSlot: def __init__(self, file, exporter, storage, uri, format, store_empty): self.file = file self.exporter = exporter @@ -203,7 +203,7 @@ class _FeedSlot(object): self._exporting = False -class FeedExporter(object): +class FeedExporter: @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 91850683f..8546628a8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -20,7 +20,7 @@ from scrapy.utils.request import request_fingerprint logger = logging.getLogger(__name__) -class DummyPolicy(object): +class DummyPolicy: def __init__(self, settings): self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') @@ -39,7 +39,7 @@ class DummyPolicy(object): return True -class RFC2616Policy(object): +class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year @@ -213,7 +213,7 @@ class RFC2616Policy(object): return currentage -class DbmCacheStorage(object): +class DbmCacheStorage: def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) @@ -270,7 +270,7 @@ class DbmCacheStorage(object): return request_fingerprint(request) -class FilesystemCacheStorage(object): +class FilesystemCacheStorage: def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR']) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index b685e7b19..0be2831a1 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -8,7 +8,7 @@ from scrapy import signals logger = logging.getLogger(__name__) -class LogStats(object): +class LogStats: """Log basic scraping stats periodically""" def __init__(self, stats, interval=60.0): diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 892aa8a86..dc8cdbb1d 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -11,7 +11,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.trackref import live_refs -class MemoryDebugger(object): +class MemoryDebugger: def __init__(self, stats): self.stats = stats diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 14e0fb32d..a0540bf8f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -19,7 +19,7 @@ from scrapy.utils.engine import get_engine_status logger = logging.getLogger(__name__) -class MemoryUsage(object): +class MemoryUsage: def __init__(self, crawler): if not crawler.settings.getbool('MEMUSAGE_ENABLED'): diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2c8e46914..2e5ff569f 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -6,7 +6,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir -class SpiderState(object): +class SpiderState: """Store and load spider state during a scraping job""" def __init__(self, jobdir=None): diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 6a982195d..320f13b29 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,7 +8,7 @@ from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured -class StatsMailer(object): +class StatsMailer: def __init__(self, stats, recipients, mail): self.stats = stats diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 198d4bbb0..56e5ad2d2 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -6,7 +6,7 @@ from scrapy import signals logger = logging.getLogger(__name__) -class AutoThrottle(object): +class AutoThrottle: def __init__(self, crawler): self.crawler = crawler diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0903fd4f8..3e810992c 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,7 +5,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode -class CookieJar(object): +class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() self.jar = _CookieJar(self.policy) @@ -100,7 +100,7 @@ def potential_domain_matches(domain): return matches + ['.' + d for d in matches] -class _DummyLock(object): +class _DummyLock: def acquire(self): pass @@ -108,7 +108,7 @@ class _DummyLock(object): pass -class WrappedRequest(object): +class WrappedRequest: """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class see http://docs.python.org/library/urllib2.html#urllib2.Request @@ -178,7 +178,7 @@ class WrappedRequest(object): self.request.headers.appendlist(name, value) -class WrappedResponse(object): +class WrappedResponse: def __init__(self, response): self.response = response diff --git a/scrapy/link.py b/scrapy/link.py index a809c5ca4..7cb0765cc 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -6,7 +6,7 @@ its documentation in: docs/topics/link-extractors.rst """ -class Link(object): +class Link: """Link objects represent an extracted link by the LinkExtractor.""" __slots__ = ['url', 'text', 'fragment', 'nofollow'] diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index bdeab3a75..6afe867b5 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -49,7 +49,7 @@ _matches = lambda url, regexs: any(r.search(url) for r in regexs) _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} -class FilteringLinkExtractor(object): +class FilteringLinkExtractor: _csstranslator = HTMLTranslator() diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index ab82e1915..fbac1dc59 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -27,7 +27,7 @@ def _nons(tag): return tag -class LxmlParserLinkExtractor(object): +class LxmlParserLinkExtractor: def __init__(self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False): self.scan_tag = tag if callable(tag) else lambda t: t == tag diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 7cf67e29e..21c4fb376 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -25,7 +25,7 @@ def unbound_method(method): return method -class ItemLoader(object): +class ItemLoader: default_item_class = Item default_input_processor = Identity() diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index 02c625acc..a7be65609 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter from scrapy.loader.common import wrap_loader_context -class MapCompose(object): +class MapCompose: def __init__(self, *functions, **default_loader_context): self.functions = functions @@ -36,7 +36,7 @@ class MapCompose(object): return values -class Compose(object): +class Compose: def __init__(self, *functions, **default_loader_context): self.functions = functions @@ -61,7 +61,7 @@ class Compose(object): return value -class TakeFirst(object): +class TakeFirst: def __call__(self, values): for value in values: @@ -69,13 +69,13 @@ class TakeFirst(object): return value -class Identity(object): +class Identity: def __call__(self, values): return values -class SelectJmes(object): +class SelectJmes: """ Query the input string for the jmespath (given at instantiation), and return the answer @@ -95,7 +95,7 @@ class SelectJmes(object): return self.compiled_path.search(value) -class Join(object): +class Join: def __init__(self, separator=u' '): self.separator = separator diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 14cec44a6..219145f13 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -14,7 +14,7 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" -class LogFormatter(object): +class LogFormatter: """Class for generating log messages for different actions. All methods must return a dictionary listing the parameters ``level``, ``msg`` diff --git a/scrapy/mail.py b/scrapy/mail.py index b2a24a3db..9d186f4f3 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -27,7 +27,7 @@ def _to_bytes_or_none(text): return to_bytes(text) -class MailSender(object): +class MailSender: def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 53fa435bb..5040378ea 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -9,7 +9,7 @@ from scrapy.utils.defer import process_parallel, process_chain, process_chain_bo logger = logging.getLogger(__name__) -class MiddlewareManager(object): +class MiddlewareManager: """Base class for implementing middleware managers""" component_name = 'foo middleware' diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 9b7445755..101bf5fbc 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ class FileException(Exception): """General media error exception""" -class FSFilesStore(object): +class FSFilesStore: def __init__(self, basedir): if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -75,7 +75,7 @@ class FSFilesStore(object): seen.add(dirname) -class S3FilesStore(object): +class S3FilesStore: AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None AWS_ENDPOINT_URL = None @@ -213,7 +213,7 @@ class S3FilesStore(object): return extra -class GCSFilesStore(object): +class GCSFilesStore: GCS_PROJECT_ID = None @@ -259,7 +259,7 @@ class GCSFilesStore(object): ) -class FTPFilesStore(object): +class FTPFilesStore: FTP_USERNAME = None FTP_PASSWORD = None diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index c174addf9..562d9ee32 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -14,11 +14,11 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -class MediaPipeline(object): +class MediaPipeline: LOG_FAILED_RESULTS = True - class SpiderInfo(object): + class SpiderInfo: def __init__(self, spider): self.spider = spider self.downloading = set() diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 1afe58dab..e13d389ee 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -110,7 +110,7 @@ class ScrapyPriorityQueue: return sum(len(x) for x in self.queues.values()) if self.queues else 0 -class DownloaderInterface(object): +class DownloaderInterface: def __init__(self, crawler): self.downloader = crawler.engine.downloader @@ -129,8 +129,8 @@ class DownloaderInterface(object): return len(self.downloader.slots[slot].active) -class DownloaderAwarePriorityQueue(object): - """ PriorityQueue which takes Downloader activity in account: +class DownloaderAwarePriorityQueue: + """ PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued first. """ diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 64bf93e86..ad89d9d22 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -11,7 +11,7 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode -class ResponseTypes(object): +class ResponseTypes: CLASSES = { 'text/html': 'scrapy.http.HtmlResponse', diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b6133619c..b9a13c018 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -28,7 +28,7 @@ def get_settings_priority(priority): return priority -class SettingsAttribute(object): +class SettingsAttribute: """Class for storing data related to settings attributes. diff --git a/scrapy/shell.py b/scrapy/shell.py index a5e140484..08ce89481 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -24,7 +24,7 @@ from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS -class Shell(object): +class Shell: relevant_classes = (Crawler, Spider, Request, Response, BaseItem, Settings) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 481d97e9a..54eb7cfa3 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -2,7 +2,7 @@ from pydispatch import dispatcher from scrapy.utils import signal as _signal -class SignalManager(object): +class SignalManager: def __init__(self, sender=dispatcher.Anonymous): self.sender = sender diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 048e84e4f..3be5aaec5 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -11,7 +11,7 @@ from scrapy.utils.spider import iter_spider_classes @implementer(ISpiderLoader) -class SpiderLoader(object): +class SpiderLoader: """ SpiderLoader is a class which locates and loads spiders in a Scrapy project. diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 34a87f2df..fa7f5bef9 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -11,7 +11,7 @@ from scrapy.http import Request logger = logging.getLogger(__name__) -class DepthMiddleware(object): +class DepthMiddleware: def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): self.maxdepth = maxdepth diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index def697c2b..375042340 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -18,7 +18,7 @@ class HttpError(IgnoreRequest): super(HttpError, self).__init__(*args, **kwargs) -class HttpErrorMiddleware(object): +class HttpErrorMiddleware: @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 2fab572e6..a006f3177 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -14,7 +14,7 @@ from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) -class OffsiteMiddleware(object): +class OffsiteMiddleware: def __init__(self, stats): self.stats = stats diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index dce2b3598..3784de885 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -28,7 +28,7 @@ POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" -class ReferrerPolicy(object): +class ReferrerPolicy: NOREFERRER_SCHEMES = LOCAL_SCHEMES @@ -284,7 +284,7 @@ def _load_policy_class(policy, warning_only=False): return None -class RefererMiddleware(object): +class RefererMiddleware: def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index a904635d8..5be1f80cb 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -12,7 +12,7 @@ from scrapy.exceptions import NotConfigured logger = logging.getLogger(__name__) -class UrlLengthMiddleware(object): +class UrlLengthMiddleware: def __init__(self, maxlength): self.maxlength = maxlength diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index a2c364c0e..d76a96451 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -34,7 +34,7 @@ def _get_method(method, spider): _default_link_extractor = LinkExtractor() -class Rule(object): +class Rule: def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None, errback=None): diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index f0bfaed34..579c60180 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -7,7 +7,7 @@ import logging logger = logging.getLogger(__name__) -class StatsCollector(object): +class StatsCollector: def __init__(self, crawler): self._dump = crawler.settings.getbool('STATS_DUMP') diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 97b5db2e1..b3e58ff94 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -8,7 +8,7 @@ from scrapy import signals -class ${ProjectName}SpiderMiddleware(object): +class ${ProjectName}SpiderMiddleware: # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @@ -56,7 +56,7 @@ class ${ProjectName}SpiderMiddleware(object): spider.logger.info('Spider opened: %s' % spider.name) -class ${ProjectName}DownloaderMiddleware(object): +class ${ProjectName}DownloaderMiddleware: # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the downloader middleware does not modify the # passed objects. diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index fb641d447..4876526a9 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -6,6 +6,6 @@ # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html -class ${ProjectName}Pipeline(object): +class ${ProjectName}Pipeline: def process_item(self, item, spider): return item diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 175f92d77..f59f4cc55 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -109,7 +109,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): return None # key is not weak-referenceable, it's not cached -class SequenceExclude(object): +class SequenceExclude: """Object to test if an item is NOT within some sequence.""" def __init__(self, seq): diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 2d3db431d..69334a918 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -144,7 +144,7 @@ def method_is_overridden(subclass, base_class, method_name): Return True if a method named ``method_name`` of a ``base_class`` is overridden in a ``subclass``. - >>> class Base(object): + >>> class Base: ... def foo(self): ... pass >>> class Sub1(Base): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7849174fb..b71419111 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -52,7 +52,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): yield xs.xpath(selxpath)[0] -class _StreamReader(object): +class _StreamReader: def __init__(self, obj): self._ptr = 0 diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index afef2c93f..5998dc33b 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -152,7 +152,7 @@ def log_scrapy_info(settings): logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) -class StreamLogger(object): +class StreamLogger: """Fake file-like stream object that redirects writes to a logger instance Taken from: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index e95a4648e..3d02d9478 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -223,7 +223,7 @@ def get_spec(func): >>> get_spec(re.match) (['pattern', 'string'], {'flags': 0}) - >>> class Test(object): + >>> class Test: ... def __call__(self, val): ... pass ... def method(self, val, flags=0): @@ -272,7 +272,7 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache(object): +class WeakKeyCache: def __init__(self, default_factory): self.default_factory = default_factory diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 17d6b2857..5308812d6 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -24,7 +24,7 @@ def listen_tcp(portrange, host, factory): raise -class CallLaterOnce(object): +class CallLaterOnce: """Schedule a function to be called in the next reactor loop, but only if it hasn't been already scheduled since the last time it ran. """ diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2f10cf4de..a57a0c291 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -10,7 +10,7 @@ from urllib.parse import urljoin import lxml.etree -class Sitemap(object): +class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 37803b287..a63c9a942 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -4,7 +4,7 @@ import os from twisted.internet import defer, protocol -class ProcessTest(object): +class ProcessTest: command = None prefix = [sys.executable, '-m', 'scrapy.cmdline'] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 9e1598805..66930ad2c 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -3,7 +3,7 @@ from urllib.parse import urljoin from twisted.web import server, resource, static, util -class SiteTest(object): +class SiteTest: def setUp(self): from twisted.internet import reactor diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 4842b95df..baed5c536 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -19,9 +19,8 @@ NoneType = type(None) live_refs = defaultdict(weakref.WeakKeyDictionary) -class object_ref(object): - """Inherit from this class (instead of object) to a keep a record of live - instances""" +class object_ref: + """Inherit from this class to a keep a record of live instances""" __slots__ = () diff --git a/tests/pipelines.py b/tests/pipelines.py index de4894c32..cf677cc17 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -3,7 +3,7 @@ Some pipelines used for testing """ -class ZeroDivisionErrorPipeline(object): +class ZeroDivisionErrorPipeline: def open_spider(self, spider): a = 1 / 0 @@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline(object): return item -class ProcessWithZeroDivisionErrorPipiline(object): +class ProcessWithZeroDivisionErrorPipiline: def process_item(self, item, spider): 1 / 0 diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index c64e87d81..6504b4d2c 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,7 +1,7 @@ """A test extension used to check the settings loading order""" -class TestExtension(object): +class TestExtension: def __init__(self, settings): settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) @@ -11,5 +11,5 @@ class TestExtension(object): return cls(crawler.settings) -class DummyExtension(object): +class DummyExtension: pass diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py index ce916f699..bd1f9cd8c 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -1,4 +1,4 @@ -class TestSpiderPipeline(object): +class TestSpiderPipeline: def open_spider(self, spider): pass @@ -7,7 +7,7 @@ class TestSpiderPipeline(object): return item -class TestSpiderExceptionPipeline(object): +class TestSpiderExceptionPipeline: def open_spider(self, spider): raise Exception('exception') diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5bf92b71a..85a24d0bc 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -89,7 +89,7 @@ class MyBadCrawlSpider(CrawlSpider): f.write(""" import logging -class MyPipeline(object): +class MyPipeline: component_name = 'my_pipeline' def process_item(self, item, spider): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 11d41c1fe..d1ce80f9d 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -25,7 +25,7 @@ class TestItem(Item): url = Field() -class ResponseMock(object): +class ResponseMock: url = 'http://scrapy.org' diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 37a069611..169e763f0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -126,7 +126,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) -class SpiderLoaderWithWrongInterface(object): +class SpiderLoaderWithWrongInterface: def unneeded_method(self): pass diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 3dd4f2351..a9190c62b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -106,7 +106,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase): def test_download_func_not_called(self): resp = Response('http://example.com/index.html') - class ResponseMiddleware(object): + class ResponseMiddleware: def process_request(self, request, spider): return resp diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 9e24d86dd..ea0e664be 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -35,7 +35,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): return df -class DirectDupeFilter(object): +class DirectDupeFilter: method = 'n/a' diff --git a/tests/test_engine.py b/tests/test_engine.py index 25dee7c1f..5b7a4e676 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -96,7 +96,7 @@ def start_test_site(debug=False): return port -class CrawlerRun(object): +class CrawlerRun: """A class to run the crawler and keep track of events occurred""" def __init__(self, spider_class): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 08e8dfc41..c5589e52f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -373,7 +373,7 @@ class StdoutFeedStorageTest(unittest.TestCase): self.assertEqual(out.getvalue(), b"content") -class FromCrawlerMixin(object): +class FromCrawlerMixin: init_with_crawler = False @classmethod diff --git a/tests/test_item.py b/tests/test_item.py index f70632d57..4017f6e84 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -213,7 +213,7 @@ class ItemTest(unittest.TestCase): class B(A): pass - class C(object): + class C: fields = {'load': Field(default='C')} not_allowed = Field(default='not_allowed') save = Field(default='C') diff --git a/tests/test_loader.py b/tests/test_loader.py index 579a85ff6..701d568dc 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -456,7 +456,7 @@ class BasicItemLoaderTest(unittest.TestCase): [u'marta', u'other'], Compose(float)) -class InitializationTestMixin(object): +class InitializationTestMixin: item_class = None diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index bf9fbe5e4..cd6cb8016 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -170,7 +170,7 @@ class SkipMessagesLogFormatter(LogFormatter): return None -class DropSomeItemsPipeline(object): +class DropSomeItemsPipeline: drop = True def process_item(self, item, spider): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index ebf817c7e..3af514bb0 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -5,7 +5,7 @@ from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager -class M1(object): +class M1: def open_spider(self, spider): pass @@ -17,7 +17,7 @@ class M1(object): pass -class M2(object): +class M2: def open_spider(self, spider): pass @@ -28,13 +28,13 @@ class M2(object): pass -class M3(object): +class M3: def process(self, response, request, spider): pass -class MOff(object): +class MOff: def open_spider(self, spider): pass diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a5cdc0de0..a3ddd50f4 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -8,7 +8,7 @@ from tests.spiders import MockServerSpider from tests.mockserver import MockServer -class InjectArgumentsDownloaderMiddleware(object): +class InjectArgumentsDownloaderMiddleware: """ Make sure downloader middlewares are able to update the keyword arguments """ @@ -23,7 +23,7 @@ class InjectArgumentsDownloaderMiddleware(object): return response -class InjectArgumentsSpiderMiddleware(object): +class InjectArgumentsSpiderMiddleware: """ Make sure spider middlewares are able to update the keyword arguments """ diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 13c297084..00568aee9 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader']) MockSlot = collections.namedtuple('MockSlot', ['active']) -class MockDownloader(object): +class MockDownloader: def __init__(self): self.slots = dict() @@ -57,7 +57,7 @@ class MockCrawler(Crawler): self.engine = MockEngine(downloader=MockDownloader()) -class SchedulerHandler(object): +class SchedulerHandler: priority_queue_cls = None jobdir = None @@ -245,7 +245,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): return True -class DownloaderAwareSchedulerTestMixin(object): +class DownloaderAwareSchedulerTestMixin: priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' reopen = False diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 7cc17600c..4c6ede70b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -47,7 +47,7 @@ class TestRefererMiddleware(TestCase): self.assertEqual(out[0].headers.get('Referer'), referrer) -class MixinDefault(object): +class MixinDefault: """ Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade @@ -72,7 +72,7 @@ class MixinDefault(object): ] -class MixinNoReferrer(object): +class MixinNoReferrer: scenarii = [ ('https://example.com/page.html', 'https://example.com/', None), ('http://www.example.com/', 'https://scrapy.org/', None), @@ -82,7 +82,7 @@ class MixinNoReferrer(object): ] -class MixinNoReferrerWhenDowngrade(object): +class MixinNoReferrerWhenDowngrade: scenarii = [ # TLS to TLS: send non-empty referrer ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), @@ -111,7 +111,7 @@ class MixinNoReferrerWhenDowngrade(object): ] -class MixinSameOrigin(object): +class MixinSameOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -144,7 +144,7 @@ class MixinSameOrigin(object): ] -class MixinOrigin(object): +class MixinOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), @@ -157,7 +157,7 @@ class MixinOrigin(object): ] -class MixinStrictOrigin(object): +class MixinStrictOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), @@ -176,7 +176,7 @@ class MixinStrictOrigin(object): ] -class MixinOriginWhenCrossOrigin(object): +class MixinOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -211,7 +211,7 @@ class MixinOriginWhenCrossOrigin(object): ] -class MixinStrictOriginWhenCrossOrigin(object): +class MixinStrictOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -255,7 +255,7 @@ class MixinStrictOriginWhenCrossOrigin(object): ] -class MixinUnsafeUrl(object): +class MixinUnsafeUrl: scenarii = [ # TLS to TLS: send referrer ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), diff --git a/tests/test_squeues.py b/tests/test_squeues.py index f0f3dd4c6..5ad8035f7 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -36,7 +36,7 @@ def nonserializable_object_test(self): self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object - class A(object): + class A: pass a = A() diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index b3a90d314..b17e17f2f 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -11,7 +11,7 @@ class MyWarning(UserWarning): pass -class SomeBaseClass(object): +class SomeBaseClass: pass @@ -155,7 +155,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class OutdatedUserClass1a(DeprecatedName): pass - class UnrelatedClass(object): + class UnrelatedClass: pass class OldStyleClass: @@ -191,7 +191,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class OutdatedUserClass2a(DeprecatedName): pass - class UnrelatedClass(object): + class UnrelatedClass: pass class OldStyleClass: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ec5b4c596..8cb8df15b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -73,7 +73,7 @@ class ToBytesTest(unittest.TestCase): class MemoizedMethodTest(unittest.TestCase): def test_memoizemethod_noargs(self): - class A(object): + class A: @memoizemethod_noargs def cached(self): @@ -153,7 +153,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): - class _Weakme(object): + class _Weakme: pass _values = count() @@ -176,14 +176,14 @@ class UtilsPythonTestCase(unittest.TestCase): def f2(a, b=None, c=None): pass - class A(object): + class A: def __init__(self, a, b, c): pass def method(self, a, b, c): pass - class Callable(object): + class Callable: def __call__(self, a, b, c): pass diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 06d9c004c..c7572f02c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -126,7 +126,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_to_dict, r) -class TestSpiderMixin(object): +class TestSpiderMixin: def __mixin_callback(self, response): pass