diff --git a/.coveragerc b/.coveragerc index 1fde07e7e..914d697a0 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,5 +4,3 @@ include = scrapy/* omit = tests/* scrapy/xlib/* - scrapy/conf.py - scrapy/log.py diff --git a/conftest.py b/conftest.py index 2d015f5e9..d8531d6cc 100644 --- a/conftest.py +++ b/conftest.py @@ -9,10 +9,6 @@ def _py_files(folder): collect_ignore = [ - # deprecated or moved modules - "scrapy/conf.py", - "scrapy/log.py", - # not a test, but looks like a test "scrapy/utils/testsite.py", diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index fa2506eb0..8d5387718 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -99,13 +99,6 @@ def execute(argv=None, settings=None): if argv is None: argv = sys.argv - # --- backward compatibility for scrapy.conf.settings singleton --- - if settings is None and 'scrapy.conf' in sys.modules: - from scrapy import conf - if hasattr(conf, 'settings'): - settings = conf.settings - # ------------------------------------------------------------------ - if settings is None: settings = get_project_settings() # set EDITOR from environment if available @@ -116,15 +109,6 @@ def execute(argv=None, settings=None): settings['EDITOR'] = editor check_deprecated_settings(settings) - # --- backward compatibility for scrapy.conf.settings singleton --- - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ScrapyDeprecationWarning) - from scrapy import conf - conf.settings = settings - # ------------------------------------------------------------------ - inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) diff --git a/scrapy/conf.py b/scrapy/conf.py deleted file mode 100644 index 6c40edcdd..000000000 --- a/scrapy/conf.py +++ /dev/null @@ -1,13 +0,0 @@ -# This module is kept for backward compatibility, so users can import -# scrapy.conf.settings and get the settings they expect - -import sys - -if 'scrapy.cmdline' not in sys.modules: - from scrapy.utils.project import get_project_settings - settings = get_project_settings() - -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.conf` is deprecated, use `crawler.settings` attribute instead", - ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4695d75f4..949dacbc8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -60,10 +60,6 @@ class Slot(object): def _get_concurrency_delay(concurrency, spider, settings): delay = settings.getfloat('DOWNLOAD_DELAY') - if hasattr(spider, 'DOWNLOAD_DELAY'): - warnings.warn("%s.DOWNLOAD_DELAY attribute is deprecated, use %s.download_delay instead" % - (type(spider).__name__, type(spider).__name__)) - delay = spider.DOWNLOAD_DELAY if hasattr(spider, 'download_delay'): delay = spider.download_delay diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index e76823623..ac4b867c3 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,15 +1,3 @@ from __future__ import absolute_import from .http10 import HTTP10DownloadHandler from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler - - -# backward compatibility -class HttpDownloadHandler(HTTP10DownloadHandler): - - def __init__(self, *args, **kwargs): - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader' - '.handlers.http10.HTTP10DownloadHandler instead', - category=ScrapyDeprecationWarning, stacklevel=1) - super(HttpDownloadHandler, self).__init__(*args, **kwargs) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 151ae4f62..f573b2b7f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -332,14 +332,7 @@ class CrawlerProcess(CrawlerRunner): def _get_spider_loader(settings): """ Get SpiderLoader instance from settings """ - if settings.get('SPIDER_MANAGER_CLASS'): - warnings.warn( - 'SPIDER_MANAGER_CLASS option is deprecated. ' - 'Please use SPIDER_LOADER_CLASS.', - category=ScrapyDeprecationWarning, stacklevel=2 - ) - cls_path = settings.get('SPIDER_MANAGER_CLASS', - settings.get('SPIDER_LOADER_CLASS')) + cls_path = settings.get('SPIDER_LOADER_CLASS') loader_cls = load_object(cls_path) try: verifyClass(ISpiderLoader, loader_cls) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 844e3828c..6665eba16 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -8,8 +8,6 @@ import six from scrapy.item import Item from scrapy.selector import Selector -from scrapy.utils.decorators import deprecated -from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import arg_to_iter, extract_regex from scrapy.utils.python import flatten @@ -191,10 +189,6 @@ class ItemLoader(object): values = self._get_xpathvalues(xpath, **kw) return self.get_value(values, *processors, **kw) - @deprecated(use_instead='._get_xpathvalues()') - def _get_values(self, xpaths, **kw): - return self._get_xpathvalues(xpaths, **kw) - def _get_xpathvalues(self, xpaths, **kw): self._check_selector_method() xpaths = arg_to_iter(xpaths) @@ -216,5 +210,3 @@ class ItemLoader(object): self._check_selector_method() csss = arg_to_iter(csss) return flatten(self.selector.css(css).getall() for css in csss) - -XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader) diff --git a/scrapy/log.py b/scrapy/log.py deleted file mode 100644 index 777bd6dc4..000000000 --- a/scrapy/log.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -This module is kept to provide a helpful warning about its removal. -""" -import logging -import warnings - -from twisted.python.failure import Failure - -from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.log import failure_to_exc_info - -logger = logging.getLogger(__name__) - -warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on " - "the builtin Python library for logging. Read the updated " - "logging entry in the documentation to learn more.", - ScrapyDeprecationWarning, stacklevel=2) - - -# Imports and level_names variable kept for backward-compatibility - -DEBUG = logging.DEBUG -INFO = logging.INFO -WARNING = logging.WARNING -ERROR = logging.ERROR -CRITICAL = logging.CRITICAL -SILENT = CRITICAL + 1 - -level_names = { - logging.DEBUG: "DEBUG", - logging.INFO: "INFO", - logging.WARNING: "WARNING", - logging.ERROR: "ERROR", - logging.CRITICAL: "CRITICAL", - SILENT: "SILENT", -} - - -def msg(message=None, _level=logging.INFO, **kw): - warnings.warn('log.msg has been deprecated, create a python logger and ' - 'log through it instead', - ScrapyDeprecationWarning, stacklevel=2) - - level = kw.pop('level', _level) - message = kw.pop('format', message) - # NOTE: logger.log doesn't handle well passing empty dictionaries with format - # arguments because of some weird use-case: - # https://hg.python.org/cpython/file/648dcafa7e5f/Lib/logging/__init__.py#l269 - logger.log(level, message, *[kw] if kw else []) - - -def err(_stuff=None, _why=None, **kw): - warnings.warn('log.err has been deprecated, create a python logger and ' - 'use its error method instead', - ScrapyDeprecationWarning, stacklevel=2) - - level = kw.pop('level', logging.ERROR) - failure = kw.pop('failure', _stuff) or Failure() - message = kw.pop('why', _why) or failure.value - logger.log(level, message, *[kw] if kw else [], exc_info=failure_to_exc_info(failure)) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 2d8091f5b..2145e6d2b 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -458,33 +458,6 @@ class FilesPipeline(MediaPipeline): return item def file_path(self, request, response=None, info=None): - ## start of deprecation warning block (can be removed in the future) - def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('FilesPipeline.file_key(url) method is deprecated, please use ' - 'file_path(request, response=None, info=None) instead', - category=ScrapyDeprecationWarning, stacklevel=1) - - # check if called from file_key with url as first argument - if not isinstance(request, Request): - _warn() - url = request - else: - url = request.url - - # detect if file_key() method has been overridden - if not hasattr(self.file_key, '_base'): - _warn() - return self.file_key(url) - ## end of deprecation warning block - - media_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation - media_ext = os.path.splitext(url)[1] # change to request.url after deprecation + media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() + media_ext = os.path.splitext(request.url)[1] return 'full/%s%s' % (media_guid, media_ext) - - # deprecated - def file_key(self, url): - return self.file_path(url) - - file_key._base = True diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index a1457c7e9..fa4d12ad1 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -165,69 +165,9 @@ class ImagesPipeline(FilesPipeline): return item def file_path(self, request, response=None, info=None): - ## start of deprecation warning block (can be removed in the future) - def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, ' - 'please use file_path(request, response=None, info=None) instead', - category=ScrapyDeprecationWarning, stacklevel=1) - - # check if called from image_key or file_key with url as first argument - if not isinstance(request, Request): - _warn() - url = request - else: - url = request.url - - # detect if file_key() or image_key() methods have been overridden - if not hasattr(self.file_key, '_base'): - _warn() - return self.file_key(url) - elif not hasattr(self.image_key, '_base'): - _warn() - return self.image_key(url) - ## end of deprecation warning block - - image_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation + image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return 'full/%s.jpg' % (image_guid) def thumb_path(self, request, thumb_id, response=None, info=None): - ## start of deprecation warning block (can be removed in the future) - def _warn(): - from scrapy.exceptions import ScrapyDeprecationWarning - import warnings - warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use ' - 'thumb_path(request, thumb_id, response=None, info=None) instead', - category=ScrapyDeprecationWarning, stacklevel=1) - - # check if called from thumb_key with url as first argument - if not isinstance(request, Request): - _warn() - url = request - else: - url = request.url - - # detect if thumb_key() method has been overridden - if not hasattr(self.thumb_key, '_base'): - _warn() - return self.thumb_key(url, thumb_id) - ## end of deprecation warning block - - thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation + thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest() return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid) - - # deprecated - def file_key(self, url): - return self.image_key(url) - file_key._base = True - - # deprecated - def image_key(self, url): - return self.file_path(url) - image_key._base = True - - # deprecated - def thumb_key(self, url, thumb_id): - return self.thumb_path(url, thumb_id) - thumb_key._base = True diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index bfbde4de9..90e96ee92 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -2,4 +2,3 @@ Selectors """ from scrapy.selector.unified import * -from scrapy.selector.lxmlsel import * diff --git a/scrapy/selector/csstranslator.py b/scrapy/selector/csstranslator.py deleted file mode 100644 index 8d7f034af..000000000 --- a/scrapy/selector/csstranslator.py +++ /dev/null @@ -1,15 +0,0 @@ -from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator -from scrapy.utils.deprecate import create_deprecated_class - - -ScrapyXPathExpr = create_deprecated_class( - 'ScrapyXPathExpr', XPathExpr, - new_class_path='parsel.csstranslator.XPathExpr') - -ScrapyGenericTranslator = create_deprecated_class( - 'ScrapyGenericTranslator', GenericTranslator, - new_class_path='parsel.csstranslator.GenericTranslator') - -ScrapyHTMLTranslator = create_deprecated_class( - 'ScrapyHTMLTranslator', HTMLTranslator, - new_class_path='parsel.csstranslator.HTMLTranslator') diff --git a/scrapy/selector/lxmlsel.py b/scrapy/selector/lxmlsel.py deleted file mode 100644 index 070cb2306..000000000 --- a/scrapy/selector/lxmlsel.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -XPath selectors based on lxml -""" -from scrapy.utils.deprecate import create_deprecated_class -from .unified import Selector, SelectorList - - -__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector', - 'XPathSelectorList'] - -def _xpathselector_css(self, *a, **kw): - raise RuntimeError('.css() method not available for %s, ' - 'instantiate scrapy.Selector ' - 'instead' % type(self).__name__) - -XPathSelector = create_deprecated_class( - 'XPathSelector', - Selector, - { - '__slots__': (), - '_default_type': 'html', - 'css': _xpathselector_css, - }, - new_class_path='scrapy.Selector', - old_class_path='scrapy.selector.XPathSelector', -) - -XmlXPathSelector = create_deprecated_class( - 'XmlXPathSelector', - XPathSelector, - clsdict={ - '__slots__': (), - '_default_type': 'xml', - }, - new_class_path='scrapy.Selector', - old_class_path='scrapy.selector.XmlXPathSelector', -) - -HtmlXPathSelector = create_deprecated_class( - 'HtmlXPathSelector', - XPathSelector, - clsdict={ - '__slots__': (), - '_default_type': 'html', - }, - new_class_path='scrapy.Selector', - old_class_path='scrapy.selector.HtmlXPathSelector', -) - -XPathSelectorList = create_deprecated_class('XPathSelectorList', SelectorList) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 8f6cb1d79..62fda40b0 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -8,7 +8,6 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.python import to_bytes from scrapy.http import HtmlResponse, XmlResponse from scrapy.utils.decorators import deprecated -from scrapy.exceptions import ScrapyDeprecationWarning __all__ = ['Selector', 'SelectorList'] @@ -31,17 +30,6 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref): The :class:`SelectorList` class is a subclass of the builtin ``list`` class, which provides a few additional methods. """ - @deprecated(use_instead='.extract()') - def extract_unquoted(self): - return [x.extract_unquoted() for x in self] - - @deprecated(use_instead='.xpath()') - def x(self, xpath): - return self.select(xpath) - - @deprecated(use_instead='.xpath()') - def select(self, xpath): - return self.xpath(xpath) class Selector(_ParselSelector, object_ref): @@ -78,21 +66,13 @@ class Selector(_ParselSelector, object_ref): __slots__ = ['response'] selectorlist_cls = SelectorList - def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs): + def __init__(self, response=None, text=None, type=None, root=None, **kwargs): if not(response is None or text is None): raise ValueError('%s.__init__() received both response and text' % self.__class__.__name__) st = _st(response, type or self._default_type) - if _root is not None: - warnings.warn("Argument `_root` is deprecated, use `root` instead", - ScrapyDeprecationWarning, stacklevel=2) - if root is None: - root = _root - else: - warnings.warn("Ignoring deprecated `_root` argument, using provided `root`") - if text is not None: response = _response_from_text(text, st) @@ -102,18 +82,3 @@ class Selector(_ParselSelector, object_ref): self.response = response super(Selector, self).__init__(text=text, type=st, root=root, **kwargs) - - # Deprecated api - @property - def _root(self): - warnings.warn("Attribute `_root` is deprecated, use `root` instead", - ScrapyDeprecationWarning, stacklevel=2) - return self.root - - @deprecated(use_instead='.xpath()') - def select(self, xpath): - return self.xpath(xpath) - - @deprecated(use_instead='.extract()') - def extract_unquoted(self): - return self.extract() diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index e9c131e3b..94095bc27 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -10,7 +10,6 @@ from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.utils.deprecate import create_deprecated_class from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import method_is_overridden @@ -52,15 +51,6 @@ class Spider(object_ref): spider._set_crawler(crawler) return spider - def set_crawler(self, crawler): - warnings.warn("set_crawler is deprecated, instantiate and bound the " - "spider to this crawler with from_crawler method " - "instead.", - category=ScrapyDeprecationWarning, stacklevel=2) - assert not hasattr(self, 'crawler'), "Spider already bounded to a " \ - "crawler" - self._set_crawler(crawler) - def _set_crawler(self, crawler): self.crawler = crawler self.settings = crawler.settings @@ -109,22 +99,6 @@ class Spider(object_ref): __repr__ = __str__ -BaseSpider = create_deprecated_class('BaseSpider', Spider) - - -class ObsoleteClass(object): - def __init__(self, message): - self.message = message - - def __getattr__(self, name): - raise AttributeError(self.message) - -spiders = ObsoleteClass( - '"from scrapy.spider import spiders" no longer works - use ' - '"from scrapy.spiderloader import SpiderLoader" and instantiate ' - 'it with your project settings"' -) - # Top-level imports from scrapy.spiders.crawl import CrawlSpider, Rule from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 6db3a1e06..90a6eb806 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -119,7 +119,3 @@ class CrawlSpider(Spider): spider._follow_links = crawler.settings.getbool( 'CRAWLSPIDER_FOLLOW_LINKS', True) return spider - - def set_crawler(self, crawler): - super(CrawlSpider, self).set_crawler(crawler) - self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True) diff --git a/scrapy/telnet.py b/scrapy/telnet.py deleted file mode 100644 index 70915f486..000000000 --- a/scrapy/telnet.py +++ /dev/null @@ -1,7 +0,0 @@ -import warnings -from scrapy.exceptions import ScrapyDeprecationWarning -warnings.warn("Module `scrapy.telnet` is deprecated, " - "use `scrapy.extensions.telnet` instead", - ScrapyDeprecationWarning, stacklevel=2) - -from scrapy.extensions.telnet import * diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index d8eaaa80f..c6140f885 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -84,19 +84,6 @@ def unique(list_, key=lambda x: x): return result -@deprecated("scrapy.utils.python.to_unicode") -def str_to_unicode(text, encoding=None, errors='strict'): - """ This function is deprecated. - Please use scrapy.utils.python.to_unicode. """ - return to_unicode(text, encoding, errors) - - -@deprecated("scrapy.utils.python.to_bytes") -def unicode_to_str(text, encoding=None, errors='strict'): - """ This function is deprecated. Please use scrapy.utils.python.to_bytes """ - return to_bytes(text, encoding, errors) - - def to_unicode(text, encoding=None, errors='strict'): """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 122af28b0..c3236afd4 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -11,14 +11,6 @@ from twisted.web import http from scrapy.utils.python import to_bytes, to_native_str from w3lib import html -from scrapy.utils.decorators import deprecated - - -@deprecated -def body_or_str(*a, **kw): - from scrapy.utils.iterators import _body_or_str - return _body_or_str(*a, **kw) - _baseurl_cache = weakref.WeakKeyDictionary() def get_base_url(response): diff --git a/tests/test_crawler.py b/tests/test_crawler.py index e811c5757..8eb2389e2 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -176,27 +176,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS']) self.assertIsInstance(spiders, sl_cls) - def test_spidermanager_deprecation(self): - with warnings.catch_warnings(record=True) as w: - runner = CrawlerRunner({ - 'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader' - }) - self.assertIsInstance(runner.spider_loader, CustomSpiderLoader) - is_one_warning = len(w) == 1 - if not is_one_warning: - for warning in w: - print(warning) - self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message)) - self.assertTrue(is_one_warning) - - def test_crawl_rejects_spider_objects(self): - with raises(ValueError): - CrawlerRunner().crawl(DefaultSpider()) - - def test_create_crawler_rejects_spider_objects(self): - with raises(ValueError): - CrawlerRunner().create_crawler(DefaultSpider()) - class CrawlerProcessTest(BaseCrawlerTest): def test_crawler_process_accepts_dict(self): diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 81235a16f..d2151e10e 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -24,7 +24,7 @@ from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler -from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler +from scrapy.core.downloader.handlers.http import HTTPDownloadHandler from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler @@ -360,11 +360,6 @@ class HttpTestCase(unittest.TestCase): return d -class DeprecatedHttpTestCase(HttpTestCase): - """HTTP 1.0 test case""" - download_handler_cls = HttpDownloadHandler - - class Http10TestCase(HttpTestCase): """HTTP 1.0 test case""" download_handler_cls = HTTP10DownloadHandler @@ -656,11 +651,6 @@ class HttpProxyTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) -class DeprecatedHttpProxyTestCase(unittest.TestCase): - """Old deprecated reference to http10 downloader handler""" - download_handler_cls = HttpDownloadHandler - - class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP10DownloadHandler diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 728a74803..0c5aaaa44 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -108,44 +108,6 @@ class FilesPipelineTestCase(unittest.TestCase): p.stop() -class DeprecatedFilesPipeline(FilesPipeline): - def file_key(self, url): - media_guid = hashlib.sha1(to_bytes(url)).hexdigest() - media_ext = os.path.splitext(url)[1] - return 'empty/%s%s' % (media_guid, media_ext) - - -class DeprecatedFilesPipelineTestCase(unittest.TestCase): - def setUp(self): - self.tempdir = mkdtemp() - - def init_pipeline(self, pipeline_class): - self.pipeline = pipeline_class.from_settings(Settings({'FILES_STORE': self.tempdir})) - self.pipeline.download_func = _mocked_download_func - self.pipeline.open_spider(None) - - def test_default_file_key_method(self): - self.init_pipeline(FilesPipeline) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.pdf"), - 'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') - self.assertEqual(len(w), 1) - self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message)) - - def test_overridden_file_key_method(self): - self.init_pipeline(DeprecatedFilesPipeline) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.pdf")), - 'empty/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf') - self.assertEqual(len(w), 1) - self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message)) - - def tearDown(self): - rmtree(self.tempdir) - - class FilesPipelineTestCaseFields(unittest.TestCase): def test_item_fields_default(self): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index a7c652959..4f7265763 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -118,63 +118,6 @@ class DeprecatedImagesPipeline(ImagesPipeline): return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid) -class DeprecatedImagesPipelineTestCase(unittest.TestCase): - def setUp(self): - self.tempdir = mkdtemp() - - def init_pipeline(self, pipeline_class): - self.pipeline = pipeline_class(self.tempdir, download_func=_mocked_download_func) - self.pipeline.open_spider(None) - - def test_default_file_key_method(self): - self.init_pipeline(ImagesPipeline) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.gif"), - 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') - self.assertEqual(len(w), 1) - self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message)) - - def test_default_image_key_method(self): - self.init_pipeline(ImagesPipeline) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self.assertEqual(self.pipeline.image_key("https://dev.mydeco.com/mydeco.gif"), - 'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') - self.assertEqual(len(w), 1) - self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message)) - - def test_overridden_file_key_method(self): - self.init_pipeline(DeprecatedImagesPipeline) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.gif")), - 'empty/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg') - self.assertEqual(len(w), 1) - self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message)) - - def test_default_thumb_key_method(self): - self.init_pipeline(ImagesPipeline) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self.assertEqual(self.pipeline.thumb_key("file:///tmp/foo.jpg", 50), - 'thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg') - self.assertEqual(len(w), 1) - self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message)) - - def test_overridden_thumb_key_method(self): - self.init_pipeline(DeprecatedImagesPipeline) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - self.assertEqual(self.pipeline.thumb_path(Request("file:///tmp/foo.jpg"), 50), - 'thumbsup/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg') - self.assertEqual(len(w), 1) - self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message)) - - def tearDown(self): - rmtree(self.tempdir) - - class ImagesPipelineTestCaseFields(unittest.TestCase): def test_item_fields_default(self): diff --git a/tests/test_selector.py b/tests/test_selector.py index bc4baf7ea..210e3256c 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -3,7 +3,6 @@ import weakref from twisted.trial import unittest from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import Selector -from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector from lxml import etree @@ -40,22 +39,6 @@ class SelectorTestCase(unittest.TestCase): sel = Selector(response) self.assertEqual(url, sel.root.base) - def test_deprecated_root_argument(self): - with warnings.catch_warnings(record=True) as w: - root = etree.fromstring(u'') - sel = Selector(_root=root) - self.assertIs(root, sel.root) - self.assertEqual(str(w[-1].message), - 'Argument `_root` is deprecated, use `root` instead') - - def test_deprecated_root_argument_ambiguous(self): - with warnings.catch_warnings(record=True) as w: - _root = etree.fromstring(u'') - root = etree.fromstring(u'') - sel = Selector(_root=_root, root=root) - self.assertIs(root, sel.root) - self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message)) - def test_flavor_detection(self): text = b'

Hello

' sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) @@ -101,111 +84,6 @@ class SelectorTestCase(unittest.TestCase): assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ x.__class__.__name__ - def test_deprecated_selector_methods(self): - sel = Selector(TextResponse(url="http://example.com", body=b'

some text

')) - - with warnings.catch_warnings(record=True) as w: - sel.select('//p') - self.assertSubstring('Use .xpath() instead', str(w[-1].message)) - - with warnings.catch_warnings(record=True) as w: - sel.extract_unquoted() - self.assertSubstring('Use .extract() instead', str(w[-1].message)) - - def test_deprecated_selectorlist_methods(self): - sel = Selector(TextResponse(url="http://example.com", body=b'

some text

')) - - with warnings.catch_warnings(record=True) as w: - sel.xpath('//p').select('.') - self.assertSubstring('Use .xpath() instead', str(w[-1].message)) - - with warnings.catch_warnings(record=True) as w: - sel.xpath('//p').extract_unquoted() - self.assertSubstring('Use .extract() instead', str(w[-1].message)) - def test_selector_bad_args(self): with self.assertRaisesRegexp(ValueError, 'received both response and text'): Selector(TextResponse(url='http://example.com', body=b''), text=u'') - - -class DeprecatedXpathSelectorTest(unittest.TestCase): - - text = '

Hello

' - - def test_warnings_xpathselector(self): - cls = XPathSelector - with warnings.catch_warnings(record=True) as w: - class UserClass(cls): - pass - - # subclassing must issue a warning - self.assertEqual(len(w), 1, str(cls)) - self.assertIn('scrapy.Selector', str(w[0].message)) - - # subclass instance doesn't issue a warning - usel = UserClass(text=self.text) - self.assertEqual(len(w), 1) - - # class instance must issue a warning - sel = cls(text=self.text) - self.assertEqual(len(w), 2, str((cls, [x.message for x in w]))) - self.assertIn('scrapy.Selector', str(w[1].message)) - - # subclass and instance checks - self.assertTrue(issubclass(cls, Selector)) - self.assertTrue(isinstance(sel, Selector)) - self.assertTrue(isinstance(usel, Selector)) - - def test_warnings_xmlxpathselector(self): - cls = XmlXPathSelector - with warnings.catch_warnings(record=True) as w: - class UserClass(cls): - pass - - # subclassing must issue a warning - self.assertEqual(len(w), 1, str(cls)) - self.assertIn('scrapy.Selector', str(w[0].message)) - - # subclass instance doesn't issue a warning - usel = UserClass(text=self.text) - self.assertEqual(len(w), 1) - - # class instance must issue a warning - sel = cls(text=self.text) - self.assertEqual(len(w), 2, str((cls, [x.message for x in w]))) - self.assertIn('scrapy.Selector', str(w[1].message)) - - # subclass and instance checks - self.assertTrue(issubclass(cls, Selector)) - self.assertTrue(issubclass(cls, XPathSelector)) - self.assertTrue(isinstance(sel, Selector)) - self.assertTrue(isinstance(usel, Selector)) - self.assertTrue(isinstance(sel, XPathSelector)) - self.assertTrue(isinstance(usel, XPathSelector)) - - def test_warnings_htmlxpathselector(self): - cls = HtmlXPathSelector - with warnings.catch_warnings(record=True) as w: - class UserClass(cls): - pass - - # subclassing must issue a warning - self.assertEqual(len(w), 1, str(cls)) - self.assertIn('scrapy.Selector', str(w[0].message)) - - # subclass instance doesn't issue a warning - usel = UserClass(text=self.text) - self.assertEqual(len(w), 1) - - # class instance must issue a warning - sel = cls(text=self.text) - self.assertEqual(len(w), 2, str((cls, [x.message for x in w]))) - self.assertIn('scrapy.Selector', str(w[1].message)) - - # subclass and instance checks - self.assertTrue(issubclass(cls, Selector)) - self.assertTrue(issubclass(cls, XPathSelector)) - self.assertTrue(isinstance(sel, Selector)) - self.assertTrue(isinstance(usel, Selector)) - self.assertTrue(isinstance(sel, XPathSelector)) - self.assertTrue(isinstance(usel, XPathSelector)) diff --git a/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py deleted file mode 100644 index 2d82fcba7..000000000 --- a/tests/test_selector_csstranslator.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Selector tests for cssselect backend -""" -import warnings -from twisted.trial import unittest -from scrapy.selector.csstranslator import ( - ScrapyHTMLTranslator, - ScrapyGenericTranslator, - ScrapyXPathExpr -) - - -class DeprecatedClassesTest(unittest.TestCase): - - def test_deprecated_warnings(self): - for cls in [ScrapyHTMLTranslator, ScrapyGenericTranslator, ScrapyXPathExpr]: - with warnings.catch_warnings(record=True) as w: - obj = cls() - self.assertIn('%s is deprecated' % cls.__name__, str(w[-1].message), - 'Missing deprecate warning for %s' % cls.__name__) - - diff --git a/tests/test_spider.py b/tests/test_spider.py index 83fb68c2f..1197ea653 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -10,7 +10,7 @@ from scrapy import signals from scrapy.settings import Settings from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse from scrapy.spiders.init import InitSpider -from scrapy.spiders import Spider, BaseSpider, CrawlSpider, Rule, XMLFeedSpider, \ +from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning @@ -51,17 +51,6 @@ class SpiderTest(unittest.TestCase): self.assertRaises(ValueError, self.spider_class) self.assertRaises(ValueError, self.spider_class, somearg='foo') - def test_deprecated_set_crawler_method(self): - spider = self.spider_class('example.com') - crawler = get_crawler() - with warnings.catch_warnings(record=True) as w: - spider.set_crawler(crawler) - self.assertIn("set_crawler", str(w[0].message)) - self.assertTrue(hasattr(spider, 'crawler')) - self.assertIs(spider.crawler, crawler) - self.assertTrue(hasattr(spider, 'settings')) - self.assertIs(spider.settings, crawler.settings) - def test_from_crawler_crawler_and_settings_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, 'example.com') @@ -377,20 +366,6 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) - def test_follow_links_attribute_deprecated_population(self): - spider = self.spider_class('example.com') - self.assertFalse(hasattr(spider, '_follow_links')) - - spider.set_crawler(get_crawler()) - self.assertTrue(hasattr(spider, '_follow_links')) - self.assertTrue(spider._follow_links) - - spider = self.spider_class('example.com') - settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False} - spider.set_crawler(get_crawler(settings_dict=settings_dict)) - self.assertTrue(hasattr(spider, '_follow_links')) - self.assertFalse(spider._follow_links) - class SitemapSpiderTest(SpiderTest): @@ -578,57 +553,9 @@ Sitemap: /sitemap-relative-url.xml class DeprecationTest(unittest.TestCase): - def test_basespider_is_deprecated(self): - with warnings.catch_warnings(record=True) as w: - - class MySpider1(BaseSpider): - pass - - self.assertEqual(len(w), 1) - self.assertEqual(w[0].category, ScrapyDeprecationWarning) - self.assertEqual(w[0].lineno, inspect.getsourcelines(MySpider1)[1]) - - def test_basespider_issubclass(self): - class MySpider2(Spider): - pass - - class MySpider2a(MySpider2): - pass - - class Foo(object): - pass - - class Foo2(object_ref): - pass - - assert issubclass(MySpider2, BaseSpider) - assert issubclass(MySpider2a, BaseSpider) - assert not issubclass(Foo, BaseSpider) - assert not issubclass(Foo2, BaseSpider) - - def test_basespider_isinstance(self): - class MySpider3(Spider): - name = 'myspider3' - - class MySpider3a(MySpider3): - pass - - class Foo(object): - pass - - class Foo2(object_ref): - pass - - assert isinstance(MySpider3(), BaseSpider) - assert isinstance(MySpider3a(), BaseSpider) - assert not isinstance(Foo(), BaseSpider) - assert not isinstance(Foo2(), BaseSpider) - def test_crawl_spider(self): assert issubclass(CrawlSpider, Spider) - assert issubclass(CrawlSpider, BaseSpider) assert isinstance(CrawlSpider(name='foo'), Spider) - assert isinstance(CrawlSpider(name='foo'), BaseSpider) def test_make_requests_from_url_deprecated(self): class MySpider4(Spider):