From b1566a696217244d2d506ce8962418943d0b1edc Mon Sep 17 00:00:00 2001 From: nyov Date: Mon, 2 Mar 2020 23:01:26 +0000 Subject: [PATCH 1/9] Remove deprecated Crawler.spiders property Deprecated since 419026615 (2014, Scrapy 0.25) --- scrapy/crawler.py | 45 ++++++++++++++++--------------------------- tests/test_crawler.py | 15 --------------- 2 files changed, 17 insertions(+), 43 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 49b8e4511..77a13d0c1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -68,17 +68,6 @@ class Crawler: self.spider = None self.engine = None - @property - def spiders(self): - if not hasattr(self, '_spiders'): - warnings.warn("Crawler.spiders is deprecated, use " - "CrawlerRunner.spider_loader or instantiate " - "scrapy.spiderloader.SpiderLoader with your " - "settings.", - category=ScrapyDeprecationWarning, stacklevel=2) - self._spiders = _get_spider_loader(self.settings.frozencopy()) - return self._spiders - @defer.inlineCallbacks def crawl(self, *args, **kwargs): assert not self.crawling, "Crawling already taking place" @@ -130,11 +119,27 @@ class CrawlerRunner: ":meth:`crawl` and managed by this class." ) + @staticmethod + def _get_spider_loader(settings): + """ Get SpiderLoader instance from settings """ + cls_path = settings.get('SPIDER_LOADER_CLASS') + loader_cls = load_object(cls_path) + try: + verifyClass(ISpiderLoader, loader_cls) + except DoesNotImplement: + warnings.warn( + 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does ' + 'not fully implement scrapy.interfaces.ISpiderLoader interface. ' + 'Please add all missing methods to avoid unexpected runtime errors.', + category=ScrapyDeprecationWarning, stacklevel=2 + ) + return loader_cls.from_settings(settings.frozencopy()) + def __init__(self, settings=None): if isinstance(settings, dict) or settings is None: settings = Settings(settings) self.settings = settings - self.spider_loader = _get_spider_loader(settings) + self.spider_loader = self._get_spider_loader(settings) self._crawlers = set() self._active = set() self.bootstrap_failed = False @@ -327,19 +332,3 @@ class CrawlerProcess(CrawlerRunner): if self.settings.get("TWISTED_REACTOR"): install_reactor(self.settings["TWISTED_REACTOR"]) super()._handle_twisted_reactor() - - -def _get_spider_loader(settings): - """ Get SpiderLoader instance from settings """ - cls_path = settings.get('SPIDER_LOADER_CLASS') - loader_cls = load_object(cls_path) - try: - verifyClass(ISpiderLoader, loader_cls) - except DoesNotImplement: - warnings.warn( - 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does ' - 'not fully implement scrapy.interfaces.ISpiderLoader interface. ' - 'Please add all missing methods to avoid unexpected runtime errors.', - category=ScrapyDeprecationWarning, stacklevel=2 - ) - return loader_cls.from_settings(settings.frozencopy()) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7bd76601d..37a069611 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -33,21 +33,6 @@ class CrawlerTestCase(BaseCrawlerTest): def setUp(self): self.crawler = Crawler(DefaultSpider, Settings()) - def test_deprecated_attribute_spiders(self): - with warnings.catch_warnings(record=True) as w: - spiders = self.crawler.spiders - self.assertEqual(len(w), 1) - self.assertIn("Crawler.spiders", str(w[0].message)) - sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS']) - self.assertIsInstance(spiders, sl_cls) - - self.crawler.spiders - is_one_warning = len(w) == 1 - if not is_one_warning: - for warning in w: - print(warning) - self.assertTrue(is_one_warning, "Warn deprecated access only once") - def test_populate_spidercls_settings(self): spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} project_settings = {'TEST1': 'project', 'TEST3': 'project'} From c2c6ea376ca2a1a0634946f95690c03fefb9990b Mon Sep 17 00:00:00 2001 From: nyov Date: Wed, 4 Mar 2020 21:30:32 +0000 Subject: [PATCH 2/9] Remove obsolete DEPRECATED_SETTINGS (deprecated.py) --- scrapy/cmdline.py | 2 -- scrapy/settings/deprecated.py | 23 ----------------------- 2 files changed, 25 deletions(-) delete mode 100644 scrapy/settings/deprecated.py diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index ec78f7c91..a4ec7c8ae 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -12,7 +12,6 @@ from scrapy.exceptions import UsageError from scrapy.utils.misc import walk_modules from scrapy.utils.project import inside_project, get_project_settings from scrapy.utils.python import garbage_collect -from scrapy.settings.deprecated import check_deprecated_settings def _iter_command_classes(module_name): @@ -118,7 +117,6 @@ def execute(argv=None, settings=None): pass else: settings['EDITOR'] = editor - check_deprecated_settings(settings) inproject = inside_project() cmds = _get_commands_dict(settings, inproject) diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py deleted file mode 100644 index f6f878725..000000000 --- a/scrapy/settings/deprecated.py +++ /dev/null @@ -1,23 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning - -DEPRECATED_SETTINGS = [ - ('TRACK_REFS', 'no longer needed (trackref is always enabled)'), - ('RESPONSE_CLASSES', 'no longer supported'), - ('DEFAULT_RESPONSE_ENCODING', 'no longer supported'), - ('BOT_VERSION', 'no longer used (user agent defaults to Scrapy now)'), - ('ENCODING_ALIASES', 'no longer needed (encoding discovery uses w3lib now)'), - ('STATS_ENABLED', 'no longer supported (change STATS_CLASS instead)'), - ('SQLITE_DB', 'no longer supported'), - ('AUTOTHROTTLE_MIN_DOWNLOAD_DELAY', 'use DOWNLOAD_DELAY instead'), - ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), -] - - -def check_deprecated_settings(settings): - deprecated = [x for x in DEPRECATED_SETTINGS if settings[x[0]] is not None] - if deprecated: - msg = "You are using the following settings which are deprecated or obsolete" - msg += " (ask scrapy-users@googlegroups.com for alternatives):" - msg = msg + "\n " + "\n ".join("%s: %s" % x for x in deprecated) - warnings.warn(msg, ScrapyDeprecationWarning) From e5711127b162ce78e6c1b60ef209792cd6eef4e6 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 16 Mar 2020 15:43:02 -0300 Subject: [PATCH 3/9] Remove deprecated ChunkedTransferMiddleware (#4431) --- scrapy/downloadermiddlewares/chunked.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 scrapy/downloadermiddlewares/chunked.py diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py deleted file mode 100644 index 6748d0265..000000000 --- a/scrapy/downloadermiddlewares/chunked.py +++ /dev/null @@ -1,21 +0,0 @@ -import warnings - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.http import decode_chunked_transfer - - -warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " - "chunked transfers are supported by default.", - ScrapyDeprecationWarning, stacklevel=2) - - -class ChunkedTransferMiddleware(object): - """This middleware adds support for chunked transfer encoding, as - documented in: https://en.wikipedia.org/wiki/Chunked_transfer_encoding - """ - - def process_response(self, request, response, spider): - if response.headers.get('Transfer-Encoding') == 'chunked': - body = decode_chunked_transfer(response.body) - return response.replace(body=body) - return response From dfbe1d95071acfbba159cea051530749b1684460 Mon Sep 17 00:00:00 2001 From: elacuesta Date: Mon, 16 Mar 2020 16:12:46 -0300 Subject: [PATCH 4/9] Remove object base class (#4430) --- docs/topics/exporters.rst | 2 +- docs/topics/extensions.rst | 2 +- docs/topics/item-pipeline.rst | 10 +++++----- docs/topics/leaks.rst | 2 +- docs/topics/settings.rst | 2 +- docs/topics/stats.rst | 2 +- scrapy/commands/__init__.py | 2 +- scrapy/commands/bench.py | 2 +- scrapy/contracts/__init__.py | 4 ++-- scrapy/core/downloader/__init__.py | 4 ++-- scrapy/core/downloader/handlers/http11.py | 4 ++-- scrapy/core/engine.py | 4 ++-- scrapy/core/scheduler.py | 2 +- scrapy/core/scraper.py | 4 ++-- scrapy/downloadermiddlewares/ajaxcrawl.py | 2 +- scrapy/downloadermiddlewares/cookies.py | 2 +- scrapy/downloadermiddlewares/decompression.py | 2 +- scrapy/downloadermiddlewares/defaultheaders.py | 2 +- .../downloadermiddlewares/downloadtimeout.py | 2 +- scrapy/downloadermiddlewares/httpauth.py | 2 +- scrapy/downloadermiddlewares/httpcache.py | 2 +- .../downloadermiddlewares/httpcompression.py | 2 +- scrapy/downloadermiddlewares/httpproxy.py | 2 +- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/downloadermiddlewares/robotstxt.py | 2 +- scrapy/downloadermiddlewares/stats.py | 2 +- scrapy/downloadermiddlewares/useragent.py | 2 +- scrapy/dupefilters.py | 2 +- scrapy/exporters.py | 2 +- scrapy/extensions/closespider.py | 2 +- scrapy/extensions/corestats.py | 2 +- scrapy/extensions/debug.py | 4 ++-- scrapy/extensions/feedexport.py | 10 +++++----- scrapy/extensions/httpcache.py | 8 ++++---- scrapy/extensions/logstats.py | 2 +- scrapy/extensions/memdebug.py | 2 +- scrapy/extensions/memusage.py | 2 +- scrapy/extensions/spiderstate.py | 2 +- scrapy/extensions/statsmailer.py | 2 +- scrapy/extensions/throttle.py | 2 +- scrapy/http/cookies.py | 8 ++++---- scrapy/link.py | 2 +- scrapy/linkextractors/__init__.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 2 +- scrapy/loader/__init__.py | 2 +- scrapy/loader/processors.py | 12 ++++++------ scrapy/logformatter.py | 2 +- scrapy/mail.py | 2 +- scrapy/middleware.py | 2 +- scrapy/pipelines/files.py | 8 ++++---- scrapy/pipelines/media.py | 4 ++-- scrapy/pqueues.py | 6 +++--- scrapy/responsetypes.py | 2 +- scrapy/settings/__init__.py | 2 +- scrapy/shell.py | 2 +- scrapy/signalmanager.py | 2 +- scrapy/spiderloader.py | 2 +- scrapy/spidermiddlewares/depth.py | 2 +- scrapy/spidermiddlewares/httperror.py | 2 +- scrapy/spidermiddlewares/offsite.py | 2 +- scrapy/spidermiddlewares/referer.py | 4 ++-- scrapy/spidermiddlewares/urllength.py | 2 +- scrapy/spiders/crawl.py | 2 +- scrapy/statscollectors.py | 2 +- .../project/module/middlewares.py.tmpl | 4 ++-- .../templates/project/module/pipelines.py.tmpl | 2 +- scrapy/utils/datatypes.py | 2 +- scrapy/utils/deprecate.py | 2 +- scrapy/utils/iterators.py | 2 +- scrapy/utils/log.py | 2 +- scrapy/utils/python.py | 4 ++-- scrapy/utils/reactor.py | 2 +- scrapy/utils/sitemap.py | 2 +- scrapy/utils/testproc.py | 2 +- scrapy/utils/testsite.py | 2 +- scrapy/utils/trackref.py | 5 ++--- tests/pipelines.py | 4 ++-- tests/test_cmdline/extensions.py | 4 ++-- .../test_spider/pipelines.py | 4 ++-- tests/test_command_parse.py | 2 +- tests/test_contracts.py | 2 +- tests/test_crawler.py | 2 +- tests/test_downloadermiddleware.py | 2 +- tests/test_dupefilters.py | 2 +- tests/test_engine.py | 2 +- tests/test_feedexport.py | 2 +- tests/test_item.py | 2 +- tests/test_loader.py | 2 +- tests/test_logformatter.py | 2 +- tests/test_middleware.py | 8 ++++---- tests/test_request_cb_kwargs.py | 4 ++-- tests/test_scheduler.py | 6 +++--- tests/test_spidermiddleware_referer.py | 18 +++++++++--------- tests/test_squeues.py | 2 +- tests/test_utils_deprecate.py | 6 +++--- tests/test_utils_python.py | 8 ++++---- tests/test_utils_reqser.py | 2 +- 98 files changed, 155 insertions(+), 156 deletions(-) diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index d411e2eed..e52682690 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -42,7 +42,7 @@ value of one of their fields:: from scrapy.exporters import XmlItemExporter - class PerYearXmlExportPipeline(object): + class PerYearXmlExportPipeline: """Distribute items across multiple XML files according to their 'year' field""" def open_spider(self, spider): diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index dc057f6b6..94fd2e36e 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -107,7 +107,7 @@ Here is the code of such extension:: logger = logging.getLogger(__name__) - class SpiderOpenCloseLogging(object): + class SpiderOpenCloseLogging: def __init__(self, item_count): self.item_count = item_count diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 98e2506e5..533f84630 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -81,7 +81,7 @@ contain a price:: from scrapy.exceptions import DropItem - class PricePipeline(object): + class PricePipeline: vat_factor = 1.15 @@ -103,7 +103,7 @@ format:: import json - class JsonWriterPipeline(object): + class JsonWriterPipeline: def open_spider(self, spider): self.file = open('items.jl', 'w') @@ -132,7 +132,7 @@ method and how to clean up the resources properly.:: import pymongo - class MongoPipeline(object): + class MongoPipeline: collection_name = 'scrapy_items' @@ -180,7 +180,7 @@ it saves the screenshot to a file and adds filename to the item. from urllib.parse import quote - class ScreenshotPipeline(object): + class ScreenshotPipeline: """Pipeline that uses Splash to render screenshot of every Scrapy item.""" @@ -219,7 +219,7 @@ returns multiples items with the same id:: from scrapy.exceptions import DropItem - class DuplicatesPipeline(object): + class DuplicatesPipeline: def __init__(self): self.ids_seen = set() diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index c0c83fc84..4ee447065 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -170,7 +170,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. .. class:: object_ref - Inherit from this class (instead of object) if you want to track live + Inherit from this class if you want to track live instances with the ``trackref`` module. .. function:: print_live_refs(class_name, ignore=NoneType) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c01202a10..dc6843d75 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -124,7 +124,7 @@ Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` attribute of the Crawler that is passed to ``from_crawler`` method in extensions, middlewares and item pipelines:: - class MyExtension(object): + class MyExtension: def __init__(self, log_is_enabled=False): if log_is_enabled: print("log is enabled!") diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 3dd829ebe..af848b402 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -32,7 +32,7 @@ Common Stats Collector uses Access the stats collector through the :attr:`~scrapy.crawler.Crawler.stats` attribute. Here is an example of an extension that access stats:: - class ExtensionThatAccessStats(object): + class ExtensionThatAccessStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 0b24193c2..a573a03d9 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -9,7 +9,7 @@ from scrapy.utils.conf import arglist_to_dict from scrapy.exceptions import UsageError -class ScrapyCommand(object): +class ScrapyCommand: requires_project = False crawler_process = None diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 7bbe362e7..c9f3b38e0 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -25,7 +25,7 @@ class Command(ScrapyCommand): self.crawler_process.start() -class _BenchServer(object): +class _BenchServer: def __enter__(self): from scrapy.utils.test import get_testenv diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 7b6591d86..41d4f25b2 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -9,7 +9,7 @@ from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_spec -class ContractsManager(object): +class ContractsManager: contracts = {} def __init__(self, contracts): @@ -107,7 +107,7 @@ class ContractsManager(object): request.errback = eb_wrapper -class Contract(object): +class Contract: """ Abstract class for contracts """ request_cls = None diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 644be121f..36aca4dae 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -13,7 +13,7 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.handlers import DownloadHandlers -class Slot(object): +class Slot: """Downloader slot""" def __init__(self, concurrency, delay, randomize_delay): @@ -66,7 +66,7 @@ def _get_concurrency_delay(concurrency, spider, settings): return concurrency, delay -class Downloader(object): +class Downloader: DOWNLOAD_SLOT = 'download_slot' diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 04a8d617a..c970909d7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -268,7 +268,7 @@ class ScrapyProxyAgent(Agent): ) -class ScrapyAgent(object): +class ScrapyAgent: _Agent = Agent _ProxyAgent = ScrapyProxyAgent @@ -432,7 +432,7 @@ class ScrapyAgent(object): @implementer(IBodyProducer) -class _RequestBodyProducer(object): +class _RequestBodyProducer: def __init__(self, body): self.body = body diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6ab8cde6b..74f03344e 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -21,7 +21,7 @@ from scrapy.utils.log import logformatter_adapter, failure_to_exc_info logger = logging.getLogger(__name__) -class Slot(object): +class Slot: def __init__(self, start_requests, close_if_idle, nextcall, scheduler): self.closing = False @@ -53,7 +53,7 @@ class Slot(object): self.closing.callback(None) -class ExecutionEngine(object): +class ExecutionEngine: def __init__(self, crawler, spider_closed_callback): self.crawler = crawler diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index c96b9b719..a18c26b17 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -14,7 +14,7 @@ from scrapy.utils.deprecate import ScrapyDeprecationWarning logger = logging.getLogger(__name__) -class Scheduler(object): +class Scheduler: """ Scrapy Scheduler. It allows to enqueue requests and then get a next request to download. Scheduler is also handling duplication diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 41f015017..3e4826216 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -21,7 +21,7 @@ from scrapy.core.spidermw import SpiderMiddlewareManager logger = logging.getLogger(__name__) -class Slot(object): +class Slot: """Scraper slot (one per running spider)""" MIN_RESPONSE_SIZE = 1024 @@ -62,7 +62,7 @@ class Slot(object): return self.active_size > self.max_active_size -class Scraper(object): +class Scraper: def __init__(self, crawler): self.slot = None diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 16b046e99..ad7a81e6b 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -11,7 +11,7 @@ from scrapy.http import HtmlResponse logger = logging.getLogger(__name__) -class AjaxCrawlMiddleware(object): +class AjaxCrawlMiddleware: """ Handle 'AJAX crawlable' pages marked as crawlable via meta tag. For more info see https://developers.google.com/webmasters/ajax-crawling/docs/getting-started. diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d8dabdf13..d57f04bc3 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -10,7 +10,7 @@ from scrapy.utils.python import to_unicode logger = logging.getLogger(__name__) -class CookiesMiddleware(object): +class CookiesMiddleware: """This middleware enables working with sites that need cookies""" def __init__(self, debug=False): diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index fcea38ef5..0fcf8fb8c 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -16,7 +16,7 @@ from scrapy.responsetypes import responsetypes logger = logging.getLogger(__name__) -class DecompressionMiddleware(object): +class DecompressionMiddleware: """ This middleware tries to recognise and extract the possibly compressed responses that may arrive. """ diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py index 93fe97673..f67961881 100644 --- a/scrapy/downloadermiddlewares/defaultheaders.py +++ b/scrapy/downloadermiddlewares/defaultheaders.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from scrapy.utils.python import without_none_values -class DefaultHeadersMiddleware(object): +class DefaultHeadersMiddleware: def __init__(self, headers): self._headers = headers diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py index 18123cfce..d373a22df 100644 --- a/scrapy/downloadermiddlewares/downloadtimeout.py +++ b/scrapy/downloadermiddlewares/downloadtimeout.py @@ -7,7 +7,7 @@ See documentation in docs/topics/downloader-middleware.rst from scrapy import signals -class DownloadTimeoutMiddleware(object): +class DownloadTimeoutMiddleware: def __init__(self, timeout=180): self._timeout = timeout diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 7aa7a62bc..089bf0d85 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -9,7 +9,7 @@ from w3lib.http import basic_auth_header from scrapy import signals -class HttpAuthMiddleware(object): +class HttpAuthMiddleware: """Set Basic HTTP Authorization header (http_user and http_pass spider class attributes)""" diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 4e06f8236..6db57bd8b 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -17,7 +17,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.utils.misc import load_object -class HttpCacheMiddleware(object): +class HttpCacheMiddleware: DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, ConnectionRefusedError, ConnectionDone, ConnectError, diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 0010b2a8f..727c41466 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -15,7 +15,7 @@ except ImportError: pass -class HttpCompressionMiddleware(object): +class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" @classmethod diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 814ce78fe..da89d3e9b 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -7,7 +7,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes -class HttpProxyMiddleware(object): +class HttpProxyMiddleware: def __init__(self, auth_encoding='latin-1'): self.auth_encoding = auth_encoding diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 08cff8a55..09ee8377e 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -11,7 +11,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured logger = logging.getLogger(__name__) -class BaseRedirectMiddleware(object): +class BaseRedirectMiddleware: enabled_setting = 'REDIRECT_ENABLED' diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 7ab5b6e62..bbf5fca05 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -25,7 +25,7 @@ from scrapy.utils.python import global_object_name logger = logging.getLogger(__name__) -class RetryMiddleware(object): +class RetryMiddleware: # IOError is raised by the HttpCompression middleware when trying to # decompress an empty response diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 251706c50..7f18b2bf2 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -16,7 +16,7 @@ from scrapy.utils.misc import load_object logger = logging.getLogger(__name__) -class RobotsTxtMiddleware(object): +class RobotsTxtMiddleware: DOWNLOAD_PRIORITY = 1000 def __init__(self, crawler): diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py index ef0aafce0..46a2ad397 100644 --- a/scrapy/downloadermiddlewares/stats.py +++ b/scrapy/downloadermiddlewares/stats.py @@ -4,7 +4,7 @@ from scrapy.utils.response import response_httprepr from scrapy.utils.python import global_object_name -class DownloaderStats(object): +class DownloaderStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py index d24750c69..3ee7bd129 100644 --- a/scrapy/downloadermiddlewares/useragent.py +++ b/scrapy/downloadermiddlewares/useragent.py @@ -3,7 +3,7 @@ from scrapy import signals -class UserAgentMiddleware(object): +class UserAgentMiddleware: """This middleware allows spiders to override the user_agent""" def __init__(self, user_agent='Scrapy'): diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index a36c8304f..d74c8ed36 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -5,7 +5,7 @@ from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint -class BaseDupeFilter(object): +class BaseDupeFilter: @classmethod def from_settings(cls, settings): diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 2e20a7180..0cb6cef98 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -21,7 +21,7 @@ __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', 'JsonItemExporter', 'MarshalItemExporter'] -class BaseItemExporter(object): +class BaseItemExporter: def __init__(self, *, dont_fail=False, **kwargs): self._kwargs = kwargs diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 260b2e86e..e3f212bef 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -10,7 +10,7 @@ from scrapy import signals from scrapy.exceptions import NotConfigured -class CloseSpider(object): +class CloseSpider: def __init__(self, crawler): self.crawler = crawler diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 20adfbe4b..389cb65bc 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -6,7 +6,7 @@ from datetime import datetime from scrapy import signals -class CoreStats(object): +class CoreStats: def __init__(self, stats): self.stats = stats diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py index 625e13249..586399784 100644 --- a/scrapy/extensions/debug.py +++ b/scrapy/extensions/debug.py @@ -17,7 +17,7 @@ from scrapy.utils.trackref import format_live_refs logger = logging.getLogger(__name__) -class StackTraceDump(object): +class StackTraceDump: def __init__(self, crawler=None): self.crawler = crawler @@ -52,7 +52,7 @@ class StackTraceDump(object): return dumps -class Debugger(object): +class Debugger: def __init__(self): try: signal.signal(signal.SIGUSR2, self._enter_debugger) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 108b6d35c..998d2a5d1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -44,7 +44,7 @@ class IFeedStorage(Interface): @implementer(IFeedStorage) -class BlockingFeedStorage(object): +class BlockingFeedStorage: def open(self, spider): path = spider.crawler.settings['FEED_TEMPDIR'] @@ -61,7 +61,7 @@ class BlockingFeedStorage(object): @implementer(IFeedStorage) -class StdoutFeedStorage(object): +class StdoutFeedStorage: def __init__(self, uri, _stdout=None): if not _stdout: @@ -76,7 +76,7 @@ class StdoutFeedStorage(object): @implementer(IFeedStorage) -class FileFeedStorage(object): +class FileFeedStorage: def __init__(self, uri): self.path = file_uri_to_path(uri) @@ -179,7 +179,7 @@ class FTPFeedStorage(BlockingFeedStorage): ) -class _FeedSlot(object): +class _FeedSlot: def __init__(self, file, exporter, storage, uri, format, store_empty): self.file = file self.exporter = exporter @@ -203,7 +203,7 @@ class _FeedSlot(object): self._exporting = False -class FeedExporter(object): +class FeedExporter: @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 91850683f..8546628a8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -20,7 +20,7 @@ from scrapy.utils.request import request_fingerprint logger = logging.getLogger(__name__) -class DummyPolicy(object): +class DummyPolicy: def __init__(self, settings): self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') @@ -39,7 +39,7 @@ class DummyPolicy(object): return True -class RFC2616Policy(object): +class RFC2616Policy: MAXAGE = 3600 * 24 * 365 # one year @@ -213,7 +213,7 @@ class RFC2616Policy(object): return currentage -class DbmCacheStorage(object): +class DbmCacheStorage: def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) @@ -270,7 +270,7 @@ class DbmCacheStorage(object): return request_fingerprint(request) -class FilesystemCacheStorage(object): +class FilesystemCacheStorage: def __init__(self, settings): self.cachedir = data_path(settings['HTTPCACHE_DIR']) diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py index b685e7b19..0be2831a1 100644 --- a/scrapy/extensions/logstats.py +++ b/scrapy/extensions/logstats.py @@ -8,7 +8,7 @@ from scrapy import signals logger = logging.getLogger(__name__) -class LogStats(object): +class LogStats: """Log basic scraping stats periodically""" def __init__(self, stats, interval=60.0): diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 892aa8a86..dc8cdbb1d 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -11,7 +11,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.trackref import live_refs -class MemoryDebugger(object): +class MemoryDebugger: def __init__(self, stats): self.stats = stats diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index 14e0fb32d..a0540bf8f 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -19,7 +19,7 @@ from scrapy.utils.engine import get_engine_status logger = logging.getLogger(__name__) -class MemoryUsage(object): +class MemoryUsage: def __init__(self, crawler): if not crawler.settings.getbool('MEMUSAGE_ENABLED'): diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2c8e46914..2e5ff569f 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -6,7 +6,7 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir -class SpiderState(object): +class SpiderState: """Store and load spider state during a scraping job""" def __init__(self, jobdir=None): diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 6a982195d..320f13b29 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -8,7 +8,7 @@ from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured -class StatsMailer(object): +class StatsMailer: def __init__(self, stats, recipients, mail): self.stats = stats diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 198d4bbb0..56e5ad2d2 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -6,7 +6,7 @@ from scrapy import signals logger = logging.getLogger(__name__) -class AutoThrottle(object): +class AutoThrottle: def __init__(self, crawler): self.crawler = crawler diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 0903fd4f8..3e810992c 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -5,7 +5,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode -class CookieJar(object): +class CookieJar: def __init__(self, policy=None, check_expired_frequency=10000): self.policy = policy or DefaultCookiePolicy() self.jar = _CookieJar(self.policy) @@ -100,7 +100,7 @@ def potential_domain_matches(domain): return matches + ['.' + d for d in matches] -class _DummyLock(object): +class _DummyLock: def acquire(self): pass @@ -108,7 +108,7 @@ class _DummyLock(object): pass -class WrappedRequest(object): +class WrappedRequest: """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class see http://docs.python.org/library/urllib2.html#urllib2.Request @@ -178,7 +178,7 @@ class WrappedRequest(object): self.request.headers.appendlist(name, value) -class WrappedResponse(object): +class WrappedResponse: def __init__(self, response): self.response = response diff --git a/scrapy/link.py b/scrapy/link.py index a809c5ca4..7cb0765cc 100644 --- a/scrapy/link.py +++ b/scrapy/link.py @@ -6,7 +6,7 @@ its documentation in: docs/topics/link-extractors.rst """ -class Link(object): +class Link: """Link objects represent an extracted link by the LinkExtractor.""" __slots__ = ['url', 'text', 'fragment', 'nofollow'] diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index bdeab3a75..6afe867b5 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -49,7 +49,7 @@ _matches = lambda url, regexs: any(r.search(url) for r in regexs) _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} -class FilteringLinkExtractor(object): +class FilteringLinkExtractor: _csstranslator = HTMLTranslator() diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index ab82e1915..fbac1dc59 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -27,7 +27,7 @@ def _nons(tag): return tag -class LxmlParserLinkExtractor(object): +class LxmlParserLinkExtractor: def __init__(self, tag="a", attr="href", process=None, unique=False, strip=True, canonicalized=False): self.scan_tag = tag if callable(tag) else lambda t: t == tag diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 7cf67e29e..21c4fb376 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -25,7 +25,7 @@ def unbound_method(method): return method -class ItemLoader(object): +class ItemLoader: default_item_class = Item default_input_processor = Identity() diff --git a/scrapy/loader/processors.py b/scrapy/loader/processors.py index 02c625acc..a7be65609 100644 --- a/scrapy/loader/processors.py +++ b/scrapy/loader/processors.py @@ -9,7 +9,7 @@ from scrapy.utils.misc import arg_to_iter from scrapy.loader.common import wrap_loader_context -class MapCompose(object): +class MapCompose: def __init__(self, *functions, **default_loader_context): self.functions = functions @@ -36,7 +36,7 @@ class MapCompose(object): return values -class Compose(object): +class Compose: def __init__(self, *functions, **default_loader_context): self.functions = functions @@ -61,7 +61,7 @@ class Compose(object): return value -class TakeFirst(object): +class TakeFirst: def __call__(self, values): for value in values: @@ -69,13 +69,13 @@ class TakeFirst(object): return value -class Identity(object): +class Identity: def __call__(self, values): return values -class SelectJmes(object): +class SelectJmes: """ Query the input string for the jmespath (given at instantiation), and return the answer @@ -95,7 +95,7 @@ class SelectJmes(object): return self.compiled_path.search(value) -class Join(object): +class Join: def __init__(self, separator=u' '): self.separator = separator diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 14cec44a6..219145f13 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -14,7 +14,7 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s" DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s" -class LogFormatter(object): +class LogFormatter: """Class for generating log messages for different actions. All methods must return a dictionary listing the parameters ``level``, ``msg`` diff --git a/scrapy/mail.py b/scrapy/mail.py index b2a24a3db..9d186f4f3 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -27,7 +27,7 @@ def _to_bytes_or_none(text): return to_bytes(text) -class MailSender(object): +class MailSender: def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost', smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False): self.smtphost = smtphost diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 53fa435bb..5040378ea 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -9,7 +9,7 @@ from scrapy.utils.defer import process_parallel, process_chain, process_chain_bo logger = logging.getLogger(__name__) -class MiddlewareManager(object): +class MiddlewareManager: """Base class for implementing middleware managers""" component_name = 'foo middleware' diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 9b7445755..101bf5fbc 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -37,7 +37,7 @@ class FileException(Exception): """General media error exception""" -class FSFilesStore(object): +class FSFilesStore: def __init__(self, basedir): if '://' in basedir: basedir = basedir.split('://', 1)[1] @@ -75,7 +75,7 @@ class FSFilesStore(object): seen.add(dirname) -class S3FilesStore(object): +class S3FilesStore: AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None AWS_ENDPOINT_URL = None @@ -213,7 +213,7 @@ class S3FilesStore(object): return extra -class GCSFilesStore(object): +class GCSFilesStore: GCS_PROJECT_ID = None @@ -259,7 +259,7 @@ class GCSFilesStore(object): ) -class FTPFilesStore(object): +class FTPFilesStore: FTP_USERNAME = None FTP_PASSWORD = None diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index c174addf9..562d9ee32 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -14,11 +14,11 @@ from scrapy.utils.log import failure_to_exc_info logger = logging.getLogger(__name__) -class MediaPipeline(object): +class MediaPipeline: LOG_FAILED_RESULTS = True - class SpiderInfo(object): + class SpiderInfo: def __init__(self, spider): self.spider = spider self.downloading = set() diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 1afe58dab..e13d389ee 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -110,7 +110,7 @@ class ScrapyPriorityQueue: return sum(len(x) for x in self.queues.values()) if self.queues else 0 -class DownloaderInterface(object): +class DownloaderInterface: def __init__(self, crawler): self.downloader = crawler.engine.downloader @@ -129,8 +129,8 @@ class DownloaderInterface(object): return len(self.downloader.slots[slot].active) -class DownloaderAwarePriorityQueue(object): - """ PriorityQueue which takes Downloader activity in account: +class DownloaderAwarePriorityQueue: + """ PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued first. """ diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 64bf93e86..ad89d9d22 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -11,7 +11,7 @@ from scrapy.utils.misc import load_object from scrapy.utils.python import binary_is_text, to_bytes, to_unicode -class ResponseTypes(object): +class ResponseTypes: CLASSES = { 'text/html': 'scrapy.http.HtmlResponse', diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index b6133619c..b9a13c018 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -28,7 +28,7 @@ def get_settings_priority(priority): return priority -class SettingsAttribute(object): +class SettingsAttribute: """Class for storing data related to settings attributes. diff --git a/scrapy/shell.py b/scrapy/shell.py index a5e140484..08ce89481 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -24,7 +24,7 @@ from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS -class Shell(object): +class Shell: relevant_classes = (Crawler, Spider, Request, Response, BaseItem, Settings) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 481d97e9a..54eb7cfa3 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -2,7 +2,7 @@ from pydispatch import dispatcher from scrapy.utils import signal as _signal -class SignalManager(object): +class SignalManager: def __init__(self, sender=dispatcher.Anonymous): self.sender = sender diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 048e84e4f..3be5aaec5 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -11,7 +11,7 @@ from scrapy.utils.spider import iter_spider_classes @implementer(ISpiderLoader) -class SpiderLoader(object): +class SpiderLoader: """ SpiderLoader is a class which locates and loads spiders in a Scrapy project. diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 34a87f2df..fa7f5bef9 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -11,7 +11,7 @@ from scrapy.http import Request logger = logging.getLogger(__name__) -class DepthMiddleware(object): +class DepthMiddleware: def __init__(self, maxdepth, stats, verbose_stats=False, prio=1): self.maxdepth = maxdepth diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index def697c2b..375042340 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -18,7 +18,7 @@ class HttpError(IgnoreRequest): super(HttpError, self).__init__(*args, **kwargs) -class HttpErrorMiddleware(object): +class HttpErrorMiddleware: @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 2fab572e6..a006f3177 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -14,7 +14,7 @@ from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) -class OffsiteMiddleware(object): +class OffsiteMiddleware: def __init__(self, stats): self.stats = stats diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index dce2b3598..3784de885 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -28,7 +28,7 @@ POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" -class ReferrerPolicy(object): +class ReferrerPolicy: NOREFERRER_SCHEMES = LOCAL_SCHEMES @@ -284,7 +284,7 @@ def _load_policy_class(policy, warning_only=False): return None -class RefererMiddleware(object): +class RefererMiddleware: def __init__(self, settings=None): self.default_policy = DefaultReferrerPolicy diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index a904635d8..5be1f80cb 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -12,7 +12,7 @@ from scrapy.exceptions import NotConfigured logger = logging.getLogger(__name__) -class UrlLengthMiddleware(object): +class UrlLengthMiddleware: def __init__(self, maxlength): self.maxlength = maxlength diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index a2c364c0e..d76a96451 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -34,7 +34,7 @@ def _get_method(method, spider): _default_link_extractor = LinkExtractor() -class Rule(object): +class Rule: def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None, errback=None): diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index f0bfaed34..579c60180 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -7,7 +7,7 @@ import logging logger = logging.getLogger(__name__) -class StatsCollector(object): +class StatsCollector: def __init__(self, crawler): self._dump = crawler.settings.getbool('STATS_DUMP') diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 97b5db2e1..b3e58ff94 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -8,7 +8,7 @@ from scrapy import signals -class ${ProjectName}SpiderMiddleware(object): +class ${ProjectName}SpiderMiddleware: # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @@ -56,7 +56,7 @@ class ${ProjectName}SpiderMiddleware(object): spider.logger.info('Spider opened: %s' % spider.name) -class ${ProjectName}DownloaderMiddleware(object): +class ${ProjectName}DownloaderMiddleware: # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the downloader middleware does not modify the # passed objects. diff --git a/scrapy/templates/project/module/pipelines.py.tmpl b/scrapy/templates/project/module/pipelines.py.tmpl index fb641d447..4876526a9 100644 --- a/scrapy/templates/project/module/pipelines.py.tmpl +++ b/scrapy/templates/project/module/pipelines.py.tmpl @@ -6,6 +6,6 @@ # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html -class ${ProjectName}Pipeline(object): +class ${ProjectName}Pipeline: def process_item(self, item, spider): return item diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 175f92d77..f59f4cc55 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -109,7 +109,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): return None # key is not weak-referenceable, it's not cached -class SequenceExclude(object): +class SequenceExclude: """Object to test if an item is NOT within some sequence.""" def __init__(self, seq): diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 2d3db431d..69334a918 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -144,7 +144,7 @@ def method_is_overridden(subclass, base_class, method_name): Return True if a method named ``method_name`` of a ``base_class`` is overridden in a ``subclass``. - >>> class Base(object): + >>> class Base: ... def foo(self): ... pass >>> class Sub1(Base): diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7849174fb..b71419111 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -52,7 +52,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): yield xs.xpath(selxpath)[0] -class _StreamReader(object): +class _StreamReader: def __init__(self, obj): self._ptr = 0 diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index afef2c93f..5998dc33b 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -152,7 +152,7 @@ def log_scrapy_info(settings): logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) -class StreamLogger(object): +class StreamLogger: """Fake file-like stream object that redirects writes to a logger instance Taken from: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index e95a4648e..3d02d9478 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -223,7 +223,7 @@ def get_spec(func): >>> get_spec(re.match) (['pattern', 'string'], {'flags': 0}) - >>> class Test(object): + >>> class Test: ... def __call__(self, val): ... pass ... def method(self, val, flags=0): @@ -272,7 +272,7 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache(object): +class WeakKeyCache: def __init__(self, default_factory): self.default_factory = default_factory diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 17d6b2857..5308812d6 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -24,7 +24,7 @@ def listen_tcp(portrange, host, factory): raise -class CallLaterOnce(object): +class CallLaterOnce: """Schedule a function to be called in the next reactor loop, but only if it hasn't been already scheduled since the last time it ran. """ diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 2f10cf4de..a57a0c291 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -10,7 +10,7 @@ from urllib.parse import urljoin import lxml.etree -class Sitemap(object): +class Sitemap: """Class to parse Sitemap (type=urlset) and Sitemap Index (type=sitemapindex) files""" diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 37803b287..a63c9a942 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -4,7 +4,7 @@ import os from twisted.internet import defer, protocol -class ProcessTest(object): +class ProcessTest: command = None prefix = [sys.executable, '-m', 'scrapy.cmdline'] diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 9e1598805..66930ad2c 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -3,7 +3,7 @@ from urllib.parse import urljoin from twisted.web import server, resource, static, util -class SiteTest(object): +class SiteTest: def setUp(self): from twisted.internet import reactor diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 4842b95df..baed5c536 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -19,9 +19,8 @@ NoneType = type(None) live_refs = defaultdict(weakref.WeakKeyDictionary) -class object_ref(object): - """Inherit from this class (instead of object) to a keep a record of live - instances""" +class object_ref: + """Inherit from this class to a keep a record of live instances""" __slots__ = () diff --git a/tests/pipelines.py b/tests/pipelines.py index de4894c32..cf677cc17 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -3,7 +3,7 @@ Some pipelines used for testing """ -class ZeroDivisionErrorPipeline(object): +class ZeroDivisionErrorPipeline: def open_spider(self, spider): a = 1 / 0 @@ -12,7 +12,7 @@ class ZeroDivisionErrorPipeline(object): return item -class ProcessWithZeroDivisionErrorPipiline(object): +class ProcessWithZeroDivisionErrorPipiline: def process_item(self, item, spider): 1 / 0 diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index c64e87d81..6504b4d2c 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,7 +1,7 @@ """A test extension used to check the settings loading order""" -class TestExtension(object): +class TestExtension: def __init__(self, settings): settings.set('TEST1', "%s + %s" % (settings['TEST1'], 'started')) @@ -11,5 +11,5 @@ class TestExtension(object): return cls(crawler.settings) -class DummyExtension(object): +class DummyExtension: pass diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py index ce916f699..bd1f9cd8c 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -1,4 +1,4 @@ -class TestSpiderPipeline(object): +class TestSpiderPipeline: def open_spider(self, spider): pass @@ -7,7 +7,7 @@ class TestSpiderPipeline(object): return item -class TestSpiderExceptionPipeline(object): +class TestSpiderExceptionPipeline: def open_spider(self, spider): raise Exception('exception') diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 5bf92b71a..85a24d0bc 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -89,7 +89,7 @@ class MyBadCrawlSpider(CrawlSpider): f.write(""" import logging -class MyPipeline(object): +class MyPipeline: component_name = 'my_pipeline' def process_item(self, item, spider): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 11d41c1fe..d1ce80f9d 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -25,7 +25,7 @@ class TestItem(Item): url = Field() -class ResponseMock(object): +class ResponseMock: url = 'http://scrapy.org' diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 37a069611..169e763f0 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -126,7 +126,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertEqual(crawler.stats.get_value('log_count/DEBUG', 0), 0) -class SpiderLoaderWithWrongInterface(object): +class SpiderLoaderWithWrongInterface: def unneeded_method(self): pass diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 3dd4f2351..a9190c62b 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -106,7 +106,7 @@ class ResponseFromProcessRequestTest(ManagerTestCase): def test_download_func_not_called(self): resp = Response('http://example.com/index.html') - class ResponseMiddleware(object): + class ResponseMiddleware: def process_request(self, request, spider): return resp diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 9e24d86dd..ea0e664be 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -35,7 +35,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): return df -class DirectDupeFilter(object): +class DirectDupeFilter: method = 'n/a' diff --git a/tests/test_engine.py b/tests/test_engine.py index 25dee7c1f..5b7a4e676 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -96,7 +96,7 @@ def start_test_site(debug=False): return port -class CrawlerRun(object): +class CrawlerRun: """A class to run the crawler and keep track of events occurred""" def __init__(self, spider_class): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 08e8dfc41..c5589e52f 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -373,7 +373,7 @@ class StdoutFeedStorageTest(unittest.TestCase): self.assertEqual(out.getvalue(), b"content") -class FromCrawlerMixin(object): +class FromCrawlerMixin: init_with_crawler = False @classmethod diff --git a/tests/test_item.py b/tests/test_item.py index f70632d57..4017f6e84 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -213,7 +213,7 @@ class ItemTest(unittest.TestCase): class B(A): pass - class C(object): + class C: fields = {'load': Field(default='C')} not_allowed = Field(default='not_allowed') save = Field(default='C') diff --git a/tests/test_loader.py b/tests/test_loader.py index 579a85ff6..701d568dc 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -456,7 +456,7 @@ class BasicItemLoaderTest(unittest.TestCase): [u'marta', u'other'], Compose(float)) -class InitializationTestMixin(object): +class InitializationTestMixin: item_class = None diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index bf9fbe5e4..cd6cb8016 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -170,7 +170,7 @@ class SkipMessagesLogFormatter(LogFormatter): return None -class DropSomeItemsPipeline(object): +class DropSomeItemsPipeline: drop = True def process_item(self, item, spider): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index ebf817c7e..3af514bb0 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -5,7 +5,7 @@ from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager -class M1(object): +class M1: def open_spider(self, spider): pass @@ -17,7 +17,7 @@ class M1(object): pass -class M2(object): +class M2: def open_spider(self, spider): pass @@ -28,13 +28,13 @@ class M2(object): pass -class M3(object): +class M3: def process(self, response, request, spider): pass -class MOff(object): +class MOff: def open_spider(self, spider): pass diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index a5cdc0de0..a3ddd50f4 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -8,7 +8,7 @@ from tests.spiders import MockServerSpider from tests.mockserver import MockServer -class InjectArgumentsDownloaderMiddleware(object): +class InjectArgumentsDownloaderMiddleware: """ Make sure downloader middlewares are able to update the keyword arguments """ @@ -23,7 +23,7 @@ class InjectArgumentsDownloaderMiddleware(object): return response -class InjectArgumentsSpiderMiddleware(object): +class InjectArgumentsSpiderMiddleware: """ Make sure spider middlewares are able to update the keyword arguments """ diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 13c297084..00568aee9 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader']) MockSlot = collections.namedtuple('MockSlot', ['active']) -class MockDownloader(object): +class MockDownloader: def __init__(self): self.slots = dict() @@ -57,7 +57,7 @@ class MockCrawler(Crawler): self.engine = MockEngine(downloader=MockDownloader()) -class SchedulerHandler(object): +class SchedulerHandler: priority_queue_cls = None jobdir = None @@ -245,7 +245,7 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): return True -class DownloaderAwareSchedulerTestMixin(object): +class DownloaderAwareSchedulerTestMixin: priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' reopen = False diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 7cc17600c..4c6ede70b 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -47,7 +47,7 @@ class TestRefererMiddleware(TestCase): self.assertEqual(out[0].headers.get('Referer'), referrer) -class MixinDefault(object): +class MixinDefault: """ Based on https://www.w3.org/TR/referrer-policy/#referrer-policy-no-referrer-when-downgrade @@ -72,7 +72,7 @@ class MixinDefault(object): ] -class MixinNoReferrer(object): +class MixinNoReferrer: scenarii = [ ('https://example.com/page.html', 'https://example.com/', None), ('http://www.example.com/', 'https://scrapy.org/', None), @@ -82,7 +82,7 @@ class MixinNoReferrer(object): ] -class MixinNoReferrerWhenDowngrade(object): +class MixinNoReferrerWhenDowngrade: scenarii = [ # TLS to TLS: send non-empty referrer ('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'), @@ -111,7 +111,7 @@ class MixinNoReferrerWhenDowngrade(object): ] -class MixinSameOrigin(object): +class MixinSameOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -144,7 +144,7 @@ class MixinSameOrigin(object): ] -class MixinOrigin(object): +class MixinOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades) ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), @@ -157,7 +157,7 @@ class MixinOrigin(object): ] -class MixinStrictOrigin(object): +class MixinStrictOrigin: scenarii = [ # TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'), @@ -176,7 +176,7 @@ class MixinStrictOrigin(object): ] -class MixinOriginWhenCrossOrigin(object): +class MixinOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -211,7 +211,7 @@ class MixinOriginWhenCrossOrigin(object): ] -class MixinStrictOriginWhenCrossOrigin(object): +class MixinStrictOriginWhenCrossOrigin: scenarii = [ # Same origin (protocol, host, port): send referrer ('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'), @@ -255,7 +255,7 @@ class MixinStrictOriginWhenCrossOrigin(object): ] -class MixinUnsafeUrl(object): +class MixinUnsafeUrl: scenarii = [ # TLS to TLS: send referrer ('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'), diff --git a/tests/test_squeues.py b/tests/test_squeues.py index f0f3dd4c6..5ad8035f7 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -36,7 +36,7 @@ def nonserializable_object_test(self): self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object - class A(object): + class A: pass a = A() diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index b3a90d314..b17e17f2f 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -11,7 +11,7 @@ class MyWarning(UserWarning): pass -class SomeBaseClass(object): +class SomeBaseClass: pass @@ -155,7 +155,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class OutdatedUserClass1a(DeprecatedName): pass - class UnrelatedClass(object): + class UnrelatedClass: pass class OldStyleClass: @@ -191,7 +191,7 @@ class WarnWhenSubclassedTest(unittest.TestCase): class OutdatedUserClass2a(DeprecatedName): pass - class UnrelatedClass(object): + class UnrelatedClass: pass class OldStyleClass: diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ec5b4c596..8cb8df15b 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -73,7 +73,7 @@ class ToBytesTest(unittest.TestCase): class MemoizedMethodTest(unittest.TestCase): def test_memoizemethod_noargs(self): - class A(object): + class A: @memoizemethod_noargs def cached(self): @@ -153,7 +153,7 @@ class UtilsPythonTestCase(unittest.TestCase): self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) def test_weakkeycache(self): - class _Weakme(object): + class _Weakme: pass _values = count() @@ -176,14 +176,14 @@ class UtilsPythonTestCase(unittest.TestCase): def f2(a, b=None, c=None): pass - class A(object): + class A: def __init__(self, a, b, c): pass def method(self, a, b, c): pass - class Callable(object): + class Callable: def __call__(self, a, b, c): pass diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 06d9c004c..c7572f02c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -126,7 +126,7 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_to_dict, r) -class TestSpiderMixin(object): +class TestSpiderMixin: def __mixin_callback(self, response): pass From 533131a30fa944688dc54dd82a581739d1ed247c Mon Sep 17 00:00:00 2001 From: sakshamb2113 <44064539+sakshamb2113@users.noreply.github.com> Date: Tue, 17 Mar 2020 14:42:49 +0530 Subject: [PATCH 5/9] Remove Guppy-specific code and documentation (#4343) --- docs/topics/leaks.rst | 64 +++---------------------------------- scrapy/extensions/telnet.py | 6 ---- 2 files changed, 5 insertions(+), 65 deletions(-) diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 4ee447065..ceb708c7e 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -17,8 +17,8 @@ what is known as a "memory leak". To help debugging memory leaks, Scrapy provides a built-in mechanism for tracking objects references called :ref:`trackref `, -and you can also use a third-party library called :ref:`Guppy -` for more advanced memory debugging (see below for more +and you can also use a third-party library called :ref:`muppy +` for more advanced memory debugging (see below for more info). Both mechanisms must be used from the :ref:`Telnet Console `. @@ -193,9 +193,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module. ``None`` if none is found. Use :func:`print_live_refs` first to get a list of all tracked live objects per class name. -.. _topics-leaks-guppy: +.. _topics-leaks-muppy: -Debugging memory leaks with Guppy +Debugging memory leaks with muppy ================================= ``trackref`` provides a very convenient mechanism for tracking down memory @@ -203,63 +203,9 @@ leaks, but it only keeps track of the objects that are more likely to cause memory leaks (Requests, Responses, Items, and Selectors). However, there are other cases where the memory leaks could come from other (more or less obscure) objects. If this is your case, and you can't find your leaks using ``trackref``, -you still have another resource: the `Guppy library`_. -If you're using Python3, see :ref:`topics-leaks-muppy`. +you still have another resource: the muppy library. -.. _Guppy library: https://pypi.org/project/guppy/ -If you use ``pip``, you can install Guppy with the following command:: - - pip install guppy - -The telnet console also comes with a built-in shortcut (``hpy``) for accessing -Guppy heap objects. Here's an example to view all Python objects available in -the heap using Guppy: - ->>> x = hpy.heap() ->>> x.bytype -Partition of a set of 297033 objects. Total size = 52587824 bytes. - Index Count % Size % Cumulative % Type - 0 22307 8 16423880 31 16423880 31 dict - 1 122285 41 12441544 24 28865424 55 str - 2 68346 23 5966696 11 34832120 66 tuple - 3 227 0 5836528 11 40668648 77 unicode - 4 2461 1 2222272 4 42890920 82 type - 5 16870 6 2024400 4 44915320 85 function - 6 13949 5 1673880 3 46589200 89 types.CodeType - 7 13422 5 1653104 3 48242304 92 list - 8 3735 1 1173680 2 49415984 94 _sre.SRE_Pattern - 9 1209 0 456936 1 49872920 95 scrapy.http.headers.Headers -<1676 more rows. Type e.g. '_.more' to view.> - -You can see that most space is used by dicts. Then, if you want to see from -which attribute those dicts are referenced, you could do: - ->>> x.bytype[0].byvia -Partition of a set of 22307 objects. Total size = 16423880 bytes. - Index Count % Size % Cumulative % Referred Via: - 0 10982 49 9416336 57 9416336 57 '.__dict__' - 1 1820 8 2681504 16 12097840 74 '.__dict__', '.func_globals' - 2 3097 14 1122904 7 13220744 80 - 3 990 4 277200 2 13497944 82 "['cookies']" - 4 987 4 276360 2 13774304 84 "['cache']" - 5 985 4 275800 2 14050104 86 "['meta']" - 6 897 4 251160 2 14301264 87 '[2]' - 7 1 0 196888 1 14498152 88 "['moduleDict']", "['modules']" - 8 672 3 188160 1 14686312 89 "['cb_kwargs']" - 9 27 0 155016 1 14841328 90 '[1]' -<333 more rows. Type e.g. '_.more' to view.> - -As you can see, the Guppy module is very powerful but also requires some deep -knowledge about Python internals. For more info about Guppy, refer to the -`Guppy documentation`_. - -.. _Guppy documentation: http://guppy-pe.sourceforge.net/ - -.. _topics-leaks-muppy: - -Debugging memory leaks with muppy -================================= You can use muppy from `Pympler`_. .. _Pympler: https://pypi.org/project/Pympler/ diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py index 26b214ee2..04ffd7235 100644 --- a/scrapy/extensions/telnet.py +++ b/scrapy/extensions/telnet.py @@ -26,11 +26,6 @@ from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp from scrapy.utils.decorators import defers -try: - import guppy - hpy = guppy.hpy() -except ImportError: - hpy = None logger = logging.getLogger(__name__) @@ -110,7 +105,6 @@ class TelnetConsole(protocol.ServerFactory): 'est': lambda: print_engine_status(self.crawler.engine), 'p': pprint.pprint, 'prefs': print_live_refs, - 'hpy': hpy, 'help': "This is Scrapy telnet console. For more info see: " "https://docs.scrapy.org/en/latest/topics/telnetconsole.html", } From 9ab45325ff6746f5c5325940fbbff1e020dd999a Mon Sep 17 00:00:00 2001 From: "Matsievskiy S.V" Date: Tue, 17 Mar 2020 18:45:00 +0300 Subject: [PATCH 6/9] edit zsh completion - Fix bug introduced in https://github.com/scrapy/scrapy/pull/4291 - Enforce `[command] [options] [arguments]` syntax. Do not allow options after arguments - Exclude already used option aliases from completion list --- extras/scrapy_zsh_completion | 76 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/extras/scrapy_zsh_completion b/extras/scrapy_zsh_completion index 33f46eda8..e2f2dc82b 100644 --- a/extras/scrapy_zsh_completion +++ b/extras/scrapy_zsh_completion @@ -14,40 +14,40 @@ _scrapy() { ;; args) case $words[1] in - bench) + (bench) _scrapy_glb_opts ;; - fetch) + (fetch) local options=( '--headers[print response HTTP headers instead of body]' '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider[use this spider]:spider:_scrapy_spiders' + '--spider=[use this spider]:spider:_scrapy_spiders' '1::URL:_httpie_urls' ) _scrapy_glb_opts $options ;; - genspider) + (genspider) local options=( - {-l,--list}'[List available templates]' - {-e,--edit}'[Edit spider after creating it]' + {'(--list)-l','(-l)--list'}'[List available templates]' + {'(--edit)-e','(-e)--edit'}'[Edit spider after creating it]' '--force[If the spider already exists, overwrite it with the template]' - {-d,--dump=}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)' - {-t,--template=}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)' + {'(--dump)-d','(-d)--dump='}'[Dump template to standard output]:template:(basic crawl csvfeed xmlfeed)' + {'(--template)-t','(-t)--template='}'[Uses a custom template]:template:(basic crawl csvfeed xmlfeed)' '1:name:(NAME)' '2:domain:_httpie_urls' ) _scrapy_glb_opts $options ;; - runspider) + (runspider) local options=( - {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)' + {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' + {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider file:_files -g \*.py' ) _scrapy_glb_opts $options ;; - settings) + (settings) local options=( '--get=[print raw setting value]:option:(SETTING)' '--getbool=[print setting value, interpreted as a boolean]:option:(SETTING)' @@ -57,77 +57,77 @@ _scrapy() { ) _scrapy_glb_opts $options ;; - shell) + (shell) local options=( '-c[evaluate the code in the shell, print the result and exit]:code:(CODE)' '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider[use this spider]:spider:_scrapy_spiders' + '--spider=[use this spider]:spider:_scrapy_spiders' '::file:_files -g \*.html' '::URL:_httpie_urls' ) _scrapy_glb_opts $options ;; - startproject) + (startproject) local options=( '1:name:(NAME)' '2:dir:_dir_list' ) _scrapy_glb_opts $options ;; - version) + (version) local options=( - {-v,--verbose}'[also display twisted/python/platform info (useful for bug reports)]' + {'(--verbose)-v','(-v)--verbose'}'[also display twisted/python/platform info (useful for bug reports)]' ) _scrapy_glb_opts $options ;; - view) + (view) local options=( '--no-redirect[do not handle HTTP 3xx status codes and print response as-is]' - '--spider[use this spider]:spider:_scrapy_spiders' + '--spider=[use this spider]:spider:_scrapy_spiders' '1:URL:_httpie_urls' ) _scrapy_glb_opts $options ;; - check) + (check) local options=( - '(- 1 *)'{-l,--list}'[only list contracts, without checking them]' - {-v,--verbose}'[print contract tests for all spiders]' + {'(--list)-l','(-l)--list'}'[only list contracts, without checking them]' + {'(--verbose)-v','(-v)--verbose'}'[print contract tests for all spiders]' '1:spider:_scrapy_spiders' ) _scrapy_glb_opts $options ;; - crawl) + (crawl) local options=( - {-o,--output}'[dump scraped items into FILE (use - for stdout)]:file:_files' - {-t,--output-format}'[format to use for dumping items with -o]:format:(FORMAT)' + {'(--output)-o','(-o)--output='}'[dump scraped items into FILE (use - for stdout)]:file:_files' + {'(--output-format)-t','(-t)--output-format='}'[format to use for dumping items with -o]:format:(FORMAT)' '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' '1:spider:_scrapy_spiders' ) _scrapy_glb_opts $options ;; - edit) + (edit) local options=( - '1:spider:_scrapy_spiders' + '1:spider:_scrapy_spiders' ) _scrapy_glb_opts $options ;; - list) + (list) _scrapy_glb_opts ;; - parse) + (parse) local options=( '*-a[set spider argument (may be repeated)]:value pair:(NAME=VALUE)' - '--spider[use this spider without looking for one]:spider:_scrapy_spiders' + '--spider=[use this spider without looking for one]:spider:_scrapy_spiders' '--pipelines[process items through pipelines]' "--nolinks[don't show links to follow (extracted requests)]" "--noitems[don't show scraped items]" '--nocolour[avoid using pygments to colorize the output]' - {-r,--rules}'[use CrawlSpider rules to discover the callback]' - {-c,--callback=}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)' - {-m,--meta=}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)' + {'(--rules)-r','(-r)--rules'}'[use CrawlSpider rules to discover the callback]' + {'(--callback)-c','(-c)--callback'}'[use this callback for parsing, instead looking for a callback]:callback:(CALLBACK)' + {'(--meta)-m','(-m)--meta='}'[inject extra meta into the Request, it must be a valid raw json string]:meta:(META)' '--cbkwargs=[inject extra callback kwargs into the Request, it must be a valid raw json string]:arguments:(CBKWARGS)' - {-d,--depth=}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)' - {-v,--verbose}'[print each depth level one by one]' + {'(--depth)-d','(-d)--depth='}'[maximum depth for parsing requests (default: 1)]:depth:(DEPTH)' + {'(--verbose)-v','(-v)--verbose'}'[print each depth level one by one]' '1:URL:_httpie_urls' ) _scrapy_glb_opts $options @@ -162,7 +162,7 @@ _scrapy_cmds() { if [[ $(scrapy -h | grep -s "no active project") == "" ]]; then commands=(${commands[@]} ${project_commands[@]}) fi - _describe -t common-commands 'common commands' commands + _describe -t common-commands 'common commands' commands && ret=0 } _scrapy_glb_opts() { @@ -172,13 +172,13 @@ _scrapy_glb_opts() { '(--nolog)--logfile=[log file. if omitted stderr will be used]:file:_files' '--pidfile=[write process ID to FILE]:file:_files' '--profile=[write python cProfile stats to FILE]:file:_files' - '(--nolog)'{-L,--loglevel=}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)' + {'(--loglevel --nolog)-L','(-L --nolog)--loglevel='}'[log level (default: INFO)]:log level:(DEBUG INFO WARN ERROR)' '(-L --loglevel --logfile)--nolog[disable logging completely]' '--pdb[enable pdb on failure]' '*'{-s,--set=}'[set/override setting (may be repeated)]:value pair:(NAME=VALUE)' ) options=(${options[@]} "$@") - _arguments $options + _arguments -A "-*" $options && ret=0 } _httpie_urls() { From 6c747953f97d60e5668cff1af3f091979ceaaa83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Mar 2020 18:33:41 +0100 Subject: [PATCH 7/9] Cover 2.0.1 in the release notes (#4437) --- docs/news.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index dd5e00223..e9b7140cd 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +.. _release-2.0.1: + +Scrapy 2.0.1 (2020-03-18) +------------------------- + +* :meth:`Response.follow_all ` now supports + an empty URL iterable as input (:issue:`4408`, :issue:`4420`) + +* Removed top-level :mod:`~twisted.internet.reactor` imports to prevent + errors about the wrong Twisted reactor being installed when setting a + different Twisted reactor using :setting:`TWISTED_REACTOR` (:issue:`4401`, + :issue:`4406`) + +* Fixed tests (:issue:`4422`) + + .. _release-2.0.0: Scrapy 2.0.0 (2020-03-03) From 532cd1d93ed58f038346ca6b753462563c85b489 Mon Sep 17 00:00:00 2001 From: Aditya Date: Fri, 20 Mar 2020 17:36:49 +0530 Subject: [PATCH 8/9] [fix] zope interface 5.0.0 unsupported --- scrapy/crawler.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 77a13d0c1..20990ea41 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,15 @@ import signal import warnings from twisted.internet import defer -from zope.interface.verify import DoesNotImplement, verifyClass +from zope.interface.exceptions import DoesNotImplement + +try: + # zope >= 5.0 only supports MultipleInvalid + from zope.interface.exceptions import MultipleInvalid +except ImportError: + MultipleInvalid = None + +from zope.interface.verify import verifyClass from scrapy import signals, Spider from scrapy.core.engine import ExecutionEngine @@ -124,9 +132,10 @@ class CrawlerRunner: """ Get SpiderLoader instance from settings """ cls_path = settings.get('SPIDER_LOADER_CLASS') loader_cls = load_object(cls_path) + excs = (DoesNotImplement, MultipleInvalid) if MultipleInvalid else DoesNotImplement try: verifyClass(ISpiderLoader, loader_cls) - except DoesNotImplement: + except excs: warnings.warn( 'SPIDER_LOADER_CLASS (previously named SPIDER_MANAGER_CLASS) does ' 'not fully implement scrapy.interfaces.ISpiderLoader interface. ' From 16f2cb4a83033b63f3adf7d7cd8b7982b8b094ef Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 26 Mar 2020 12:57:39 +0000 Subject: [PATCH 9/9] project URLs in machine-readable format for showing in pypi --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index 85d797f88..1b3c6771a 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,11 @@ setup( name='Scrapy', version=version, url='https://scrapy.org', + project_urls = { + 'Documentation': 'https://docs.scrapy.org/', + 'Source': 'https://github.com/scrapy/scrapy', + 'Tracker': 'https://github.com/scrapy/scrapy/issues', + }, description='A high-level Web Crawling and Web Scraping framework', long_description=open('README.rst').read(), author='Scrapy developers',