From d8251332845d48d2418f0055c27686cee04b5b9a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 7 Jun 2025 01:59:09 +0500 Subject: [PATCH] Reduce deps on unittest, unify inlineCallbacks imports in tests. (#6873) --- tests/test_closespider.py | 16 +- tests/test_command_fetch.py | 10 +- tests/test_command_parse.py | 42 ++-- tests/test_command_runspider.py | 5 +- tests/test_command_shell.py | 39 ++-- tests/test_command_version.py | 6 +- tests/test_contracts.py | 4 +- tests/test_crawl.py | 135 +++++++------ tests/test_dependencies.py | 12 -- tests/test_downloader_handlers.py | 2 +- tests/test_downloader_handlers_http_base.py | 12 +- ...st_downloadermiddleware_httpcompression.py | 35 ++-- tests/test_downloadermiddleware_robotstxt.py | 8 +- tests/test_downloaderslotssettings.py | 4 +- tests/test_engine.py | 24 +-- tests/test_engine_stop_download_bytes.py | 4 +- tests/test_engine_stop_download_headers.py | 4 +- tests/test_exporters.py | 3 +- tests/test_extension_periodic_log.py | 3 +- tests/test_extension_telnet.py | 12 +- tests/test_feedexport.py | 179 +++++++++--------- tests/test_http2_client_protocol.py | 4 +- tests/test_logformatter.py | 6 +- tests/test_pipeline_crawl.py | 12 +- tests/test_pipeline_files.py | 22 +-- tests/test_pipelines.py | 13 +- tests/test_proxy_connect.py | 8 +- tests/test_request_attribute_binding.py | 16 +- tests/test_request_cb_kwargs.py | 4 +- tests/test_request_left.py | 10 +- tests/test_scheduler.py | 6 +- tests/test_scheduler_base.py | 5 +- tests/test_signals.py | 4 +- tests/test_spidermiddleware.py | 15 +- tests/test_spidermiddleware_httperror.py | 8 +- tests/test_spidermiddleware_output_chain.py | 26 +-- tests/test_squeues_request.py | 13 +- tests/test_utils_log.py | 13 +- tests/test_utils_signal.py | 3 +- tests/test_utils_trackref.py | 4 +- tests/test_utils_url.py | 3 +- tests/test_webclient.py | 35 ++-- 42 files changed, 380 insertions(+), 409 deletions(-) diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 4a17b254b..c6ec690a1 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,4 +1,4 @@ -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy.utils.test import get_crawler @@ -22,7 +22,7 @@ class TestCloseSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_itemcount(self): close_on = 5 crawler = get_crawler(ItemSpider, {"CLOSESPIDER_ITEMCOUNT": close_on}) @@ -32,7 +32,7 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") assert itemcount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount(self): close_on = 5 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_PAGECOUNT": close_on}) @@ -42,7 +42,7 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") assert pagecount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount_no_item(self): close_on = 5 max_items = 5 @@ -62,7 +62,7 @@ class TestCloseSpider(TestCase): itemcount = crawler.stats.get_value("item_scraped_count") assert pagecount <= close_on + itemcount - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_pagecount_no_item_with_pagecount(self): close_on_pagecount_no_item = 5 close_on_pagecount = 20 @@ -79,7 +79,7 @@ class TestCloseSpider(TestCase): pagecount = crawler.stats.get_value("response_received_count") assert pagecount < close_on_pagecount - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_errorcount(self): close_on = 5 crawler = get_crawler(ErrorSpider, {"CLOSESPIDER_ERRORCOUNT": close_on}) @@ -91,7 +91,7 @@ class TestCloseSpider(TestCase): assert crawler.stats.get_value("spider_exceptions/count") >= close_on assert errorcount >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_timeout(self): close_on = 0.1 crawler = get_crawler(FollowAllSpider, {"CLOSESPIDER_TIMEOUT": close_on}) @@ -101,7 +101,7 @@ class TestCloseSpider(TestCase): total_seconds = crawler.stats.get_value("elapsed_time_seconds") assert total_seconds >= close_on - @defer.inlineCallbacks + @inlineCallbacks def test_closespider_timeout_no_item(self): timeout = 1 crawler = get_crawler(SlowSpider, {"CLOSESPIDER_TIMEOUT_NO_ITEM": timeout}) diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index a31cada85..89f664336 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,4 +1,4 @@ -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from tests.utils.testproc import ProcessTest @@ -8,17 +8,17 @@ from tests.utils.testsite import SiteTest class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): command = "fetch" - @defer.inlineCallbacks + @inlineCallbacks def test_output(self): _, out, _ = yield self.execute([self.url("/text")]) assert out.strip() == b"Works" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_default(self): _, out, _ = yield self.execute([self.url("/redirect")]) assert out.strip() == b"Redirected here" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_disabled(self): _, out, err = yield self.execute( ["--no-redirect", self.url("/redirect-no-meta-refresh")] @@ -27,7 +27,7 @@ class TestFetchCommand(ProcessTest, SiteTest, unittest.TestCase): assert b"downloader/response_status_count/302" in err, err assert b"downloader/response_status_count/200" not in err, err - @defer.inlineCallbacks + @inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url("/text"), "--headers"]) out = out.replace(b"\r", b"") # required on win32 diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index 9e66d319c..6681aba17 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -3,7 +3,7 @@ import os import re from pathlib import Path -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.commands import parse from scrapy.settings import Settings @@ -171,7 +171,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """ ) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_arguments(self): _, _, stderr = yield self.execute( [ @@ -187,7 +187,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_request_with_meta(self): raw_json_string = '{"foo" : "baz"}' _, _, stderr = yield self.execute( @@ -218,7 +218,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_request_with_cb_kwargs(self): raw_json_string = '{"foo" : "bar", "key": "value"}' _, _, stderr = yield self.execute( @@ -239,7 +239,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} "DEBUG: request.callback signature: (response, foo=None, key=None)" in log ) - @defer.inlineCallbacks + @inlineCallbacks def test_request_without_meta(self): _, _, stderr = yield self.execute( [ @@ -253,7 +253,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_pipelines(self): _, _, stderr = yield self.execute( [ @@ -268,7 +268,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "INFO: It Works!" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): status, out, stderr = yield self.execute( [ @@ -283,7 +283,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert "{'id': 1}" in _textmode(out) assert "{'id': 2}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): status, out, stderr = yield self.execute( [ @@ -297,7 +297,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert "INFO: Got response 200" in _textmode(stderr) assert "{'foo': 42}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): status, out, stderr = yield self.execute( [ @@ -312,7 +312,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} for i in range(10): assert f"{{'foo': {i}}}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): status, out, stderr = yield self.execute( [ @@ -327,7 +327,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} for i in range(7): assert f"{{'foo': {i}}}" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse(self): _, _, stderr = yield self.execute( [ @@ -340,21 +340,21 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: Got response 200" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_parse_items(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "parse", self.url("/html")] ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_parse_items_no_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, self.url("/html")] ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_wrong_callback_passed(self): status, out, stderr = yield self.execute( ["--spider", self.spider_name, "-c", "dummy", self.url("/html")] @@ -362,7 +362,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "Cannot find callback" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_matching_rule_callback_set(self): """If a rule matches the URL, use it's defined callback.""" status, out, stderr = yield self.execute( @@ -370,7 +370,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'foo': 'bar'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_matching_rule_default_callback(self): """If a rule match but it has no callback set, use the 'parse' callback.""" status, out, stderr = yield self.execute( @@ -378,7 +378,7 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "[{}, {'nomatch': 'default'}]" in _textmode(out) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_with_no_rules_attribute(self): """Using -r with a spider with no rule should not produce items.""" status, out, stderr = yield self.execute( @@ -387,14 +387,14 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "No CrawlSpider rules found" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_missing_callback(self): status, out, stderr = yield self.execute( ["--spider", "badcrawl" + self.spider_name, "-r", self.url("/html")] ) assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_no_matching_rule(self): """The requested URL has no matching rule, so no items should be scraped""" status, out, stderr = yield self.execute( @@ -403,12 +403,12 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} assert re.search(r"# Scraped Items -+\n\[\]", _textmode(out)) assert "Cannot find a rule that matches" in _textmode(stderr) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_not_exists_with_not_matched_url(self): status, out, stderr = yield self.execute([self.url("/invalid_url")]) assert status == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_output_flag(self): """Checks if a file was created successfully having correct format containing correct data in it. diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index c57c09249..7f8d9fb61 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING from unittest import skipIf import pytest -from twisted.trial import unittest from tests.test_commands import TestCommandBase from tests.test_crawler import ExceptionSpider, NoRequestsSpider @@ -376,7 +375,7 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): def setUp(self): if platform.system() != "Windows": - raise unittest.SkipTest("Windows required for .pyw files") + pytest.skip("Windows required for .pyw files") return super().setUp() def test_start_errors(self): @@ -385,4 +384,4 @@ class TestWindowsRunSpiderCommand(TestRunSpiderCommand): assert "badspider.pyw" in log def test_runspider_unable_to_load(self): - raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") + pytest.skip("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 8041e7cb1..d9f17d76b 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -3,8 +3,9 @@ import sys from io import BytesIO from pathlib import Path +import pytest from pexpect.popen_spawn import PopenSpawn -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.utils.reactor import _asyncio_reactor_path @@ -17,52 +18,52 @@ from tests.utils.testsite import SiteTest class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): command = "shell" - @defer.inlineCallbacks + @inlineCallbacks def test_empty(self): _, out, _ = yield self.execute(["-c", "item"]) assert b"{}" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_body(self): _, out, _ = yield self.execute([self.url("/text"), "-c", "response.body"]) assert b"Works" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_type_text(self): _, out, _ = yield self.execute([self.url("/text"), "-c", "type(response)"]) assert b"TextResponse" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_type_html(self): _, out, _ = yield self.execute([self.url("/html"), "-c", "type(response)"]) assert b"HtmlResponse" in out - @defer.inlineCallbacks + @inlineCallbacks def test_response_selector_html(self): xpath = "response.xpath(\"//p[@class='one']/text()\").get()" _, out, _ = yield self.execute([self.url("/html"), "-c", xpath]) assert out.strip() == b"Works" - @defer.inlineCallbacks + @inlineCallbacks def test_response_encoding_gb18030(self): _, out, _ = yield self.execute( [self.url("/enc-gb18030"), "-c", "response.encoding"] ) assert out.strip() == b"gb18030" - @defer.inlineCallbacks + @inlineCallbacks def test_redirect(self): _, out, _ = yield self.execute([self.url("/redirect"), "-c", "response.url"]) assert out.strip().endswith(b"/redirected") - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_follow_302(self): _, out, _ = yield self.execute( [self.url("/redirect-no-meta-refresh"), "-c", "response.status"] ) assert out.strip().endswith(b"200") - @defer.inlineCallbacks + @inlineCallbacks def test_redirect_not_follow_302(self): _, out, _ = yield self.execute( [ @@ -74,7 +75,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): ) assert out.strip().endswith(b"302") - @defer.inlineCallbacks + @inlineCallbacks def test_fetch_redirect_follow_302(self): """Test that calling ``fetch(url)`` follows HTTP redirects by default.""" url = self.url("/redirect-no-meta-refresh") @@ -84,7 +85,7 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert b"Redirecting (302)" in errout assert b"Crawled (200)" in errout - @defer.inlineCallbacks + @inlineCallbacks def test_fetch_redirect_not_follow_302(self): """Test that calling ``fetch(url, redirect=False)`` disables automatic redirects.""" url = self.url("/redirect-no-meta-refresh") @@ -93,27 +94,27 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert errcode == 0, out assert b"Crawled (302)" in errout - @defer.inlineCallbacks + @inlineCallbacks def test_request_replace(self): url = self.url("/text") code = f"fetch('{url}') or fetch(response.request.replace(method='POST'))" errcode, out, _ = yield self.execute(["-c", code]) assert errcode == 0, out - @defer.inlineCallbacks + @inlineCallbacks def test_scrapy_import(self): url = self.url("/text") code = f"fetch(scrapy.Request('{url}'))" errcode, out, _ = yield self.execute(["-c", code]) assert errcode == 0, out - @defer.inlineCallbacks + @inlineCallbacks def test_local_file(self): filepath = Path(tests_datadir, "test_site", "index.html") _, out, _ = yield self.execute([str(filepath), "-c", "item"]) assert b"{}" in out - @defer.inlineCallbacks + @inlineCallbacks def test_local_nofile(self): filepath = "file:///tests/sample_data/test_site/nothinghere.html" errcode, out, err = yield self.execute( @@ -122,16 +123,16 @@ class TestShellCommand(ProcessTest, SiteTest, unittest.TestCase): assert errcode == 1, out or err assert b"No such file or directory" in err - @defer.inlineCallbacks + @inlineCallbacks def test_dns_failures(self): if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") url = "www.somedomainthatdoesntexi.st" errcode, out, err = yield self.execute([url, "-c", "item"], check_code=False) assert errcode == 1, out or err assert b"DNS lookup failed" in err - @defer.inlineCallbacks + @inlineCallbacks def test_shell_fetch_async(self): url = self.url("/html") code = f"fetch('{url}')" diff --git a/tests/test_command_version.py b/tests/test_command_version.py index a61a6a32b..87dfb16df 100644 --- a/tests/test_command_version.py +++ b/tests/test_command_version.py @@ -1,6 +1,6 @@ import sys -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest import scrapy @@ -10,13 +10,13 @@ from tests.utils.testproc import ProcessTest class TestVersionCommand(ProcessTest, unittest.TestCase): command = "version" - @defer.inlineCallbacks + @inlineCallbacks def test_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute([]) assert out.strip().decode(encoding) == f"Scrapy {scrapy.__version__}" - @defer.inlineCallbacks + @inlineCallbacks def test_verbose_output(self): encoding = sys.stdout.encoding or "utf-8" _, out, _ = yield self.execute(["-v"]) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 26b16a1d4..ad3efa042 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,7 +1,7 @@ from unittest import TextTestResult import pytest -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.python import failure from twisted.trial import unittest @@ -502,7 +502,7 @@ class TestContractsManager(unittest.TestCase): assert not self.results.failures assert self.results.errors - @defer.inlineCallbacks + @inlineCallbacks def test_same_url(self): class TestSameUrlSpider(Spider): name = "test_same_url" diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8289b2243..4c1f6216b 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -2,7 +2,6 @@ from __future__ import annotations import json import logging -import unittest from ipaddress import IPv4Address from socket import gethostbyname from typing import Any @@ -10,7 +9,7 @@ from urllib.parse import urlparse import pytest from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -67,21 +66,21 @@ class TestCrawl(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def test_follow_all(self): crawler = get_crawler(FollowAllSpider) yield crawler.crawl(mockserver=self.mockserver) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url - @defer.inlineCallbacks + @inlineCallbacks def test_fixed_delay(self): yield self._test_delay(total=3, delay=0.2) - @defer.inlineCallbacks + @inlineCallbacks def test_randomized_delay(self): yield self._test_delay(total=3, delay=0.1, randomize=True) - @defer.inlineCallbacks + @inlineCallbacks def _test_delay(self, total, delay, randomize=False): crawl_kwargs = { "maxlatency": delay * 2, @@ -110,7 +109,7 @@ class TestCrawl(TestCase): average = total_time / (len(times) - 1) assert average <= delay / tolerance, "test total or delay values are too small" - @defer.inlineCallbacks + @inlineCallbacks def test_timeout_success(self): crawler = get_crawler(DelaySpider) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -118,7 +117,7 @@ class TestCrawl(TestCase): assert crawler.spider.t2 > 0 assert crawler.spider.t2 > crawler.spider.t1 - @defer.inlineCallbacks + @inlineCallbacks def test_timeout_failure(self): crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) yield crawler.crawl(n=0.5, mockserver=self.mockserver) @@ -135,7 +134,7 @@ class TestCrawl(TestCase): assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > crawler.spider.t1 - @defer.inlineCallbacks + @inlineCallbacks def test_retry_503(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -144,7 +143,7 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_failed(self): crawler = get_crawler(SimpleSpider) with LogCapture() as log: @@ -153,10 +152,10 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_dns_error(self): if NON_EXISTING_RESOLVABLE: - raise unittest.SkipTest("Non-existing hosts are resolvable") + pytest.skip("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) with LogCapture() as log: # try to fetch the homepage of a nonexistent domain @@ -165,7 +164,7 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -176,7 +175,7 @@ class TestCrawl(TestCase): assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks + @inlineCallbacks def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(BrokenStartSpider) @@ -187,7 +186,7 @@ class TestCrawl(TestCase): assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks + @inlineCallbacks def test_start_items(self): items = [] @@ -202,7 +201,7 @@ class TestCrawl(TestCase): assert len(log.records) == 0 assert items == [{"name": "test item"}] - @defer.inlineCallbacks + @inlineCallbacks def test_start_unsupported_output(self): """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item(), and letting @@ -223,7 +222,7 @@ class TestCrawl(TestCase): assert len(items) == 3 assert not any(isinstance(item, Request) for item in items) - @defer.inlineCallbacks + @inlineCallbacks def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartSpider, settings) @@ -241,7 +240,7 @@ class TestCrawl(TestCase): ) assert crawler.spider.visited == 3 - @defer.inlineCallbacks + @inlineCallbacks def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" @@ -275,7 +274,7 @@ with multiples lines ) assert str(log).count("Got response 200") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_lost(self): # connection lost after receiving data crawler = get_crawler(SimpleSpider) @@ -285,7 +284,7 @@ with multiples lines ) self._assert_retried(log) - @defer.inlineCallbacks + @inlineCallbacks def test_retry_conn_aborted(self): # connection lost before receiving data crawler = get_crawler(SimpleSpider) @@ -299,7 +298,7 @@ with multiples lines assert str(log).count("Retrying") == 2 assert str(log).count("Gave up retrying") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_referer_header(self): """Referer header is set by RefererMiddleware unless it is already set""" req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) @@ -327,7 +326,7 @@ with multiples lines echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) assert echo3["headers"].get("Referer") == ["http://example.com"] - @defer.inlineCallbacks + @inlineCallbacks def test_engine_status(self): from scrapy.utils.engine import get_engine_status @@ -345,7 +344,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_format_engine_status(self): from scrapy.utils.engine import format_engine_status @@ -370,7 +369,7 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @defer.inlineCallbacks + @inlineCallbacks def test_open_spider_error_on_faulty_pipeline(self): settings = { "ITEM_PIPELINES": { @@ -378,15 +377,13 @@ with multiples lines } } crawler = get_crawler(SimpleSpider, settings) - yield self.assertFailure( - crawler.crawl( + with pytest.raises(ZeroDivisionError): + yield crawler.crawl( self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ), - ZeroDivisionError, - ) + ) assert not crawler.crawling - @defer.inlineCallbacks + @inlineCallbacks def test_crawlerrunner_accepts_crawler(self): crawler = get_crawler(SimpleSpider) runner = CrawlerRunner() @@ -398,7 +395,7 @@ with multiples lines ) assert "Got response 200" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawl_multiple(self): runner = CrawlerRunner(get_reactor_settings()) runner.crawl( @@ -431,7 +428,7 @@ class TestCrawlSpider(TestCase): def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - @defer.inlineCallbacks + @inlineCallbacks def _run_spider(self, spider_cls): items = [] @@ -446,7 +443,7 @@ class TestCrawlSpider(TestCase): ) return log, items, crawler.stats - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_parse(self): crawler = get_crawler(CrawlSpiderWithParseMethod) with LogCapture() as log: @@ -456,7 +453,7 @@ class TestCrawlSpider(TestCase): assert "[parse] status 201 (foo: None)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_async_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncCallback) with LogCapture() as log: @@ -466,7 +463,7 @@ class TestCrawlSpider(TestCase): assert "[parse_async] status 201 (foo: None)" in str(log) assert "[parse_async] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_async_generator_callback(self): crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) with LogCapture() as log: @@ -476,7 +473,7 @@ class TestCrawlSpider(TestCase): assert "[parse_async_gen] status 201 (foo: None)" in str(log) assert "[parse_async_gen] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_with_errback(self): crawler = get_crawler(CrawlSpiderWithErrback) with LogCapture() as log: @@ -489,7 +486,7 @@ class TestCrawlSpider(TestCase): assert "[errback] status 500" in str(log) assert "[errback] status 501" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_crawlspider_process_request_cb_kwargs(self): crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) with LogCapture() as log: @@ -499,7 +496,7 @@ class TestCrawlSpider(TestCase): assert "[parse] status 201 (foo: process_request)" in str(log) assert "[parse] status 202 (foo: bar)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_parse(self): crawler = get_crawler(AsyncDefSpider) with LogCapture() as log: @@ -509,7 +506,7 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse(self): crawler = get_crawler( AsyncDefAsyncioSpider, @@ -524,7 +521,7 @@ class TestCrawlSpider(TestCase): assert "Got response 200" in str(log) @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_list(self): log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider) assert "Got response 200" in str(log) @@ -532,7 +529,7 @@ class TestCrawlSpider(TestCase): assert {"id": 2} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_items_single_element(self): items = [] @@ -549,7 +546,7 @@ class TestCrawlSpider(TestCase): assert {"foo": 42} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse(self): log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider) assert "Got response 200" in str(log) @@ -557,7 +554,7 @@ class TestCrawlSpider(TestCase): assert itemcount == 1 @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_loop(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider) assert "Got response 200" in str(log) @@ -567,7 +564,7 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_exc(self): log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider) log = str(log) @@ -579,7 +576,7 @@ class TestCrawlSpider(TestCase): assert {"foo": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncgen_parse_complex(self): _, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider) itemcount = stats.get_value("item_scraped_count") @@ -591,37 +588,37 @@ class TestCrawlSpider(TestCase): assert {"index2": i} in items @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_asyncio_parse_reqs_list(self): log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider) for req_id in range(3): assert f"Got response 200, req_id {req_id}" in str(log) @pytest.mark.only_not_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_direct(self): _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider) assert items == [{"code": 200}] @pytest.mark.only_asyncio - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_wrapped(self): log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider) assert items == [{"code": 200}] - @defer.inlineCallbacks + @inlineCallbacks def test_async_def_deferred_maybe_wrapped(self): _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider) assert items == [{"code": 200}] - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=False) yield crawler.crawl(seed=url, mockserver=self.mockserver) assert crawler.spider.meta["responses"][0].certificate is None - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test", is_secure=True) @@ -634,7 +631,7 @@ class TestCrawlSpider(TestCase): @pytest.mark.xfail( reason="Responses with no body return early and contain no certificate" ) - @defer.inlineCallbacks + @inlineCallbacks def test_response_ssl_certificate_empty_response(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200", is_secure=True) @@ -644,7 +641,7 @@ class TestCrawlSpider(TestCase): assert cert.getSubject().commonName == b"localhost" assert cert.getIssuer().commonName == b"localhost" - @defer.inlineCallbacks + @inlineCallbacks def test_dns_server_ip_address_none(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/status?n=200") @@ -652,7 +649,7 @@ class TestCrawlSpider(TestCase): ip_address = crawler.spider.meta["responses"][0].ip_address assert ip_address is None - @defer.inlineCallbacks + @inlineCallbacks def test_dns_server_ip_address(self): crawler = get_crawler(SingleRequestSpider) url = self.mockserver.url("/echo?body=test") @@ -662,7 +659,7 @@ class TestCrawlSpider(TestCase): assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) - @defer.inlineCallbacks + @inlineCallbacks def test_bytes_received_stop_download_callback(self): crawler = get_crawler(BytesReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -676,7 +673,7 @@ class TestCrawlSpider(TestCase): < crawler.spider.full_response_length ) - @defer.inlineCallbacks + @inlineCallbacks def test_bytes_received_stop_download_errback(self): crawler = get_crawler(BytesReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -692,7 +689,7 @@ class TestCrawlSpider(TestCase): < crawler.spider.full_response_length ) - @defer.inlineCallbacks + @inlineCallbacks def test_headers_received_stop_download_callback(self): crawler = get_crawler(HeadersReceivedCallbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -702,7 +699,7 @@ class TestCrawlSpider(TestCase): "headers_received" ) - @defer.inlineCallbacks + @inlineCallbacks def test_headers_received_stop_download_errback(self): crawler = get_crawler(HeadersReceivedErrbackSpider) yield crawler.crawl(mockserver=self.mockserver) @@ -714,7 +711,7 @@ class TestCrawlSpider(TestCase): "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback(self): failures = [] @@ -731,7 +728,7 @@ class TestCrawlSpider(TestCase): assert "HTTP status code is not handled or not allowed" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_silence(self): failures = [] @@ -747,7 +744,7 @@ class TestCrawlSpider(TestCase): assert "HTTP status code is not handled or not allowed" not in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -759,7 +756,7 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -773,7 +770,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -787,7 +784,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -804,7 +801,7 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -817,7 +814,7 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_item(self): def eb(failure: Failure) -> Any: return {"foo": "bar"} @@ -831,7 +828,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "'item_scraped_count': 1" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_spider_errback_downloader_error_request(self): def eb(failure: Failure) -> Request: return Request(self.mockserver.url("/")) @@ -845,7 +842,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) assert "Crawled (200)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_raise_closespider(self): def cb(response): raise CloseSpider @@ -856,7 +853,7 @@ class TestCrawlSpider(TestCase): assert "Closing spider (cancelled)" in str(log) assert "Spider error processing" not in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_raise_closespider_reason(self): def cb(response): raise CloseSpider("my_reason") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index c2df67c66..4436efd9b 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,25 +1,13 @@ import os import re from configparser import ConfigParser -from importlib import import_module from pathlib import Path import pytest from twisted import version as twisted_version -from twisted.trial import unittest class TestScrapyUtils: - def test_required_openssl_version(self): - try: - module = import_module("OpenSSL") - except ImportError: - raise unittest.SkipTest("OpenSSL is not available") - - if hasattr(module, "__version__"): - installed_version = [int(x) for x in module.__version__.split(".")[:2]] - assert installed_version >= [0, 6], "OpenSSL >= 0.6 required" - def test_pinned_twisted_version(self): """When running tests within a Tox environment with pinned dependencies, make sure that the version of Twisted is the pinned diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 09cdbaf35..2c8e96040 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -440,7 +440,7 @@ class TestFTPBase(unittest.TestCase): class TestFTP(TestFTPBase): def test_invalid_credentials(self): if self.reactor_pytest != "default" and sys.platform == "win32": - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) from twisted.protocols.ftp import ConnectionLost diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 14e12a3e6..9b2c49fd4 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -14,7 +14,7 @@ from unittest import mock import pytest from testfixtures import LogCapture from twisted.internet import defer, error -from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import inlineCallbacks, maybeDeferred from twisted.protocols.policies import WrappingFactory from twisted.trial import unittest from twisted.web import resource, server, static, util @@ -186,7 +186,7 @@ class TestHttpBase(unittest.TestCase, ABC): self.download_handler_cls, get_crawler() ) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): @@ -229,7 +229,7 @@ class TestHttpBase(unittest.TestCase, ABC): async def test_timeout_download_from_spider_nodata_rcvd(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) @@ -245,7 +245,7 @@ class TestHttpBase(unittest.TestCase, ABC): async def test_timeout_download_from_spider_server_hangs(self): if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 - raise unittest.SkipTest( + pytest.skip( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) # client connects, server send headers and some body bytes but hangs @@ -531,7 +531,7 @@ class TestSimpleHttpsBase(unittest.TestCase, ABC): crawler = get_crawler(settings_dict=settings_dict) self.download_handler = build_from_crawler(self.download_handler_cls, crawler) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): @@ -665,7 +665,7 @@ class TestHttpProxyBase(unittest.TestCase, ABC): self.download_handler_cls, get_crawler() ) - @defer.inlineCallbacks + @inlineCallbacks def tearDown(self): yield self.port.stopListening() if hasattr(self.download_handler, "close"): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index e7427c5ac..3c26b242f 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -2,7 +2,6 @@ from gzip import GzipFile from io import BytesIO from logging import WARNING from pathlib import Path -from unittest import SkipTest import pytest from testfixtures import LogCapture @@ -130,7 +129,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") response = self._getresponse("br") request = response.request assert response.headers["Content-Encoding"] == b"br" @@ -146,11 +145,11 @@ class TestHttpCompression: try: import brotli # noqa: F401 - raise SkipTest("Requires not having brotli support") + pytest.skip("Requires not having brotli support") except ImportError: import brotlicffi # noqa: F401 - raise SkipTest("Requires not having brotli support") + pytest.skip("Requires not having brotli support") except ImportError: pass response = self._getresponse("br") @@ -180,7 +179,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -201,7 +200,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 - raise SkipTest("Requires not having zstandard support") + pytest.skip("Requires not having zstandard support") except ImportError: pass response = self._getresponse("zstd-static-content-size") @@ -520,7 +519,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_setting("br") def test_compression_bomb_setting_deflate(self): @@ -533,7 +532,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_setting("zstd") def _test_compression_bomb_spider_attr(self, compression_id): @@ -556,7 +555,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_spider_attr("br") def test_compression_bomb_spider_attr_deflate(self): @@ -569,7 +568,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_spider_attr("zstd") def _test_compression_bomb_request_meta(self, compression_id): @@ -590,7 +589,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_compression_bomb_request_meta("br") def test_compression_bomb_request_meta_deflate(self): @@ -603,7 +602,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_compression_bomb_request_meta("zstd") def _test_download_warnsize_setting(self, compression_id): @@ -639,7 +638,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_setting("br") def test_download_warnsize_setting_deflate(self): @@ -652,7 +651,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_setting("zstd") def _test_download_warnsize_spider_attr(self, compression_id): @@ -690,7 +689,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_spider_attr("br") def test_download_warnsize_spider_attr_deflate(self): @@ -703,7 +702,7 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_spider_attr("zstd") def _test_download_warnsize_request_meta(self, compression_id): @@ -739,7 +738,7 @@ class TestHttpCompression: except ImportError: import brotlicffi # noqa: F401 except ImportError: - raise SkipTest("no brotli") + pytest.skip("no brotli") self._test_download_warnsize_request_meta("br") def test_download_warnsize_request_meta_deflate(self): @@ -752,5 +751,5 @@ class TestHttpCompression: try: import zstandard # noqa: F401 except ImportError: - raise SkipTest("no zstd support (zstandard)") + pytest.skip("no zstd support (zstandard)") self._test_download_warnsize_request_meta("zstd") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 04800896c..146b0057e 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -235,12 +235,10 @@ Disallow: /some/randome/page.html self, request: Request, middleware: RobotsTxtMiddleware ) -> None: spider = None # not actually used - await maybe_deferred_to_future( - self.assertFailure( - middleware.process_request(request, spider), # type: ignore[arg-type] - IgnoreRequest, + with pytest.raises(IgnoreRequest): + await maybe_deferred_to_future( + maybeDeferred(middleware.process_request, request, spider) # type: ignore[call-overload] ) - ) def assertRobotsTxtRequested(self, base_url: str) -> None: calls = self.crawler.engine.download.call_args_list diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 78c83ea83..9b7c09448 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,6 +1,6 @@ import time -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from scrapy import Request @@ -62,7 +62,7 @@ class CrawlTestCase(TestCase): def setUp(self): self.runner = CrawlerRunner() - @defer.inlineCallbacks + @inlineCallbacks def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_engine.py b/tests/test_engine.py index 9f618437c..e181a36cf 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -26,6 +26,7 @@ import pytest from itemadapter import ItemAdapter from pydispatch import dispatcher from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from twisted.web import server, static, util @@ -390,7 +391,7 @@ class TestEngineBase(unittest.TestCase): class TestEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, @@ -407,20 +408,20 @@ class TestEngine(TestEngineBase): self._assert_signals_caught(run) self._assert_bytes_received(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_dupefilter(self): run = CrawlerRun(DupeFilterSpider) yield run.run() self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_itemerror(self): run = CrawlerRun(ItemZeroDivisionErrorSpider) yield run.run() self._assert_items_error(run) - @defer.inlineCallbacks + @inlineCallbacks def test_crawler_change_close_reason_on_idle(self): run = CrawlerRun(ChangeCloseReasonSpider) yield run.run() @@ -429,7 +430,7 @@ class TestEngine(TestEngineBase): "reason": "custom_reason", } == run.signals_caught[signals.spider_closed] - @defer.inlineCallbacks + @inlineCallbacks def test_close_downloader(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.close() @@ -447,19 +448,14 @@ class TestEngine(TestEngineBase): get_crawler(MySpider, {"DOWNLOADER": BadDownloader}), lambda _: None ) - @defer.inlineCallbacks + @inlineCallbacks def test_start_already_running_exception(self): e = ExecutionEngine(get_crawler(MySpider), lambda _: None) yield e.open_spider(MySpider(), []) e.start() - - def cb(exc: BaseException) -> None: - assert str(exc), "Engine already running" - - try: - yield self.assertFailure(e.start(), RuntimeError).addBoth(cb) - finally: - yield e.stop() + with pytest.raises(RuntimeError, match="Engine already running"): + yield e.start() + yield e.stop() def test_short_timeout(self): args = ( diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index f09b0e091..2662e45e1 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -1,5 +1,5 @@ from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -19,7 +19,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): class TestBytesReceivedEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index dbb0ea0d2..142715927 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -1,5 +1,5 @@ from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from scrapy.exceptions import StopDownload from tests.test_engine import ( @@ -19,7 +19,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): class TestHeadersReceivedEngine(TestEngineBase): - @defer.inlineCallbacks + @inlineCallbacks def test_crawler(self): for spider in ( MySpider, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index f55cb6c97..05e8865bc 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,7 +4,6 @@ import marshal import pickle import re import tempfile -import unittest from datetime import datetime from io import BytesIO from typing import Any @@ -662,7 +661,7 @@ class TestCustomExporterItem: def setup_method(self): if self.item_class is None: - raise unittest.SkipTest("item class is None") + pytest.skip("item class is None") def test_exporter_custom_serializer(self): class CustomItemExporter(BaseItemExporter): diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 85bd42857..b86f3c7f2 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,7 +1,6 @@ from __future__ import annotations import datetime -import unittest from typing import Any, Callable from scrapy.extensions.periodic_log import PeriodicLog @@ -66,7 +65,7 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: return CustomPeriodicLog.from_crawler(crawler) -class TestPeriodicLog(unittest.TestCase): +class TestPeriodicLog: def test_extension_enabled(self): # Expected that settings for this extension loaded successfully # And on certain conditions - extension raising NotConfigured diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 8c897c223..2ac4d7830 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,6 +1,7 @@ +import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.extensions.telnet import TelnetConsole @@ -21,15 +22,16 @@ class TelnetExtensionTest(unittest.TestCase): return console, portal - @defer.inlineCallbacks + @inlineCallbacks def test_bad_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword(b"username", b"password") d = portal.login(creds, None, ITelnetProtocol) - yield self.assertFailure(d, ValueError) + with pytest.raises(ValueError, match="Invalid credentials"): + yield d console.stop_listening() - @defer.inlineCallbacks + @inlineCallbacks def test_good_credentials(self): console, portal = self._get_console_and_portal() creds = credentials.UsernamePassword( @@ -39,7 +41,7 @@ class TelnetExtensionTest(unittest.TestCase): yield d console.stop_listening() - @defer.inlineCallbacks + @inlineCallbacks def test_custom_credentials(self): settings = { "TELNETCONSOLE_USERNAME": "user", diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 44cd10ec3..cdf03ca76 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,6 +26,7 @@ import lxml.etree import pytest from testfixtures import LogCapture from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import file_uri_to_path, path_to_file_uri from zope.interface import implementer @@ -131,7 +132,7 @@ class TestFileFeedStorage(unittest.TestCase): FileFeedStorage(str(path), feed_options={"overwrite": True}), path ) - @defer.inlineCallbacks + @inlineCallbacks def _assert_stores(self, storage, path: Path, expected_content=b"content"): spider = scrapy.Spider("default") file = storage.open(spider) @@ -172,7 +173,7 @@ class TestFTPFeedStorage(unittest.TestCase): finally: path.unlink() - @defer.inlineCallbacks + @inlineCallbacks def test_append(self): with MockFTPServer() as ftp_server: filename = "file" @@ -182,7 +183,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar", feed_options=feed_options) self._assert_stored(ftp_server.path / filename, b"foobar") - @defer.inlineCallbacks + @inlineCallbacks def test_overwrite(self): with MockFTPServer() as ftp_server: filename = "file" @@ -191,7 +192,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar") self._assert_stored(ftp_server.path / filename, b"bar") - @defer.inlineCallbacks + @inlineCallbacks def test_append_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -202,7 +203,7 @@ class TestFTPFeedStorage(unittest.TestCase): yield self._store(url, b"bar", feed_options=feed_options, settings=settings) self._assert_stored(ftp_server.path / filename, b"foobar") - @defer.inlineCallbacks + @inlineCallbacks def test_overwrite_active_mode(self): with MockFTPServer() as ftp_server: settings = {"FEED_STORAGE_FTP_ACTIVE": True} @@ -290,7 +291,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.access_key == "uri_key" assert storage.secret_key == "uri_secret" - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): settings = { "AWS_ACCESS_KEY_ID": "access_key", @@ -431,7 +432,7 @@ class TestS3FeedStorage(unittest.TestCase): assert storage.region_name == region_name assert storage.s3_client._client_config.region_name == region_name - @defer.inlineCallbacks + @inlineCallbacks def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", @@ -451,7 +452,7 @@ class TestS3FeedStorage(unittest.TestCase): ) assert acl is None - @defer.inlineCallbacks + @inlineCallbacks def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" @@ -489,7 +490,7 @@ class TestGCSFeedStorage(unittest.TestCase): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"} crawler = get_crawler(settings_dict=settings) @@ -503,7 +504,7 @@ class TestGCSFeedStorage(unittest.TestCase): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""} crawler = get_crawler(settings_dict=settings) @@ -515,12 +516,12 @@ class TestGCSFeedStorage(unittest.TestCase): storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv") assert storage.acl is None - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): try: from google.cloud.storage import Client # noqa: F401 except ImportError: - raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage") + pytest.skip("GCSFeedStorage requires google-cloud-storage") uri = "gs://mybucket/export.csv" project_id = "myproject-123" @@ -556,7 +557,7 @@ class TestGCSFeedStorage(unittest.TestCase): class TestStdoutFeedStorage(unittest.TestCase): - @defer.inlineCallbacks + @inlineCallbacks def test_store(self): out = BytesIO() storage = StdoutFeedStorage("stdout:", _stdout=out) @@ -669,7 +670,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): def tearDown(self): shutil.rmtree(self.temp_dir, ignore_errors=True) - @defer.inlineCallbacks + @inlineCallbacks def exported_data(self, items, settings): """ Return exported data which a spider yielding ``items`` would return. @@ -684,7 +685,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) return data - @defer.inlineCallbacks + @inlineCallbacks def exported_no_data(self, settings): """ Return exported data which a spider yielding no ``items`` would return. @@ -699,7 +700,7 @@ class TestFeedExportBase(ABC, unittest.TestCase): data = yield self.run_and_export(TestSpider, settings) return data - @defer.inlineCallbacks + @inlineCallbacks def assertExported(self, items, header, rows, settings=None): yield self.assertExportedCsv(items, header, rows, settings) yield self.assertExportedJsonLines(items, rows, settings) @@ -770,7 +771,7 @@ class ExceptionJsonItemExporter(JsonItemExporter): class TestFeedExport(TestFeedExportBase): - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -800,7 +801,7 @@ class TestFeedExport(TestFeedExportBase): return content - @defer.inlineCallbacks + @inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( @@ -815,7 +816,7 @@ class TestFeedExport(TestFeedExportBase): assert reader.fieldnames == list(header) assert rows == list(reader) - @defer.inlineCallbacks + @inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -830,7 +831,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{k: v for k, v in row.items() if v} for row in rows] assert rows == parsed - @defer.inlineCallbacks + @inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -846,7 +847,7 @@ class TestFeedExport(TestFeedExportBase): got_rows = [{e.tag: e.text for e in it} for it in root.findall("item")] assert rows == got_rows - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -867,7 +868,7 @@ class TestFeedExport(TestFeedExportBase): json_rows = json.loads(to_unicode(data["json"])) assert rows == json_rows - @defer.inlineCallbacks + @inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -884,7 +885,7 @@ class TestFeedExport(TestFeedExportBase): result = self._load_until_eof(data["pickle"], load_func=pickle.load) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -901,7 +902,7 @@ class TestFeedExport(TestFeedExportBase): result = self._load_until_eof(data["marshal"], load_func=marshal.load) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def test_stats_file_success(self): settings = { "FEEDS": { @@ -915,7 +916,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_stats_file_failed(self): settings = { "FEEDS": { @@ -933,7 +934,7 @@ class TestFeedExport(TestFeedExportBase): assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats() assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1 - @defer.inlineCallbacks + @inlineCallbacks def test_stats_multiple_file(self): settings = { "FEEDS": { @@ -955,7 +956,7 @@ class TestFeedExport(TestFeedExportBase): crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage") == 1 ) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items(self): # feed exporters use field names from Item items = [ @@ -969,7 +970,7 @@ class TestFeedExport(TestFeedExportBase): header = self.MyItem.fields.keys() yield self.assertExported(items, header, rows) - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -981,7 +982,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert data[fmt] is None - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_items(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1001,7 +1002,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_no_items(self): items = [] settings = { @@ -1019,7 +1020,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_items_exception(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1040,7 +1041,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_start_finish_exporting_no_items_exception(self): items = [] settings = { @@ -1059,7 +1060,7 @@ class TestFeedExport(TestFeedExportBase): assert not listener.start_without_finish assert not listener.finish_without_start - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -1079,7 +1080,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert expctd == data[fmt] - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_multiple_feeds(self): """Make sure that `storage.store` is called for every feed.""" settings = { @@ -1097,7 +1098,7 @@ class TestFeedExport(TestFeedExportBase): assert str(log).count("Storage.store is called") == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1119,7 +1120,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, header, rows_csv) yield self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_empty_field_list(self): # FEED_EXPORT_FIELDS==[] means the same as default None items = [{"foo": "bar"}] @@ -1129,7 +1130,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, header, rows) yield self.assertExportedJsonLines(items, rows, settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1137,7 +1138,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_comma_separated_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1145,7 +1146,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": ",".join(header)} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_json_field_list(self): items = [{"foo": "bar"}] header = ["foo", "baz"] @@ -1153,7 +1154,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} yield self.assertExported(items, header, rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1161,7 +1162,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_dict_field_names(self): items = [{"foo": "bar"}] header = { @@ -1172,7 +1173,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": header} yield self.assertExported(items, ["Baz", "Foo"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_items_json_field_names(self): items = [{"foo": "bar"}] header = {"foo": "Foo"} @@ -1180,7 +1181,7 @@ class TestFeedExport(TestFeedExportBase): settings = {"FEED_EXPORT_FIELDS": json.dumps(header)} yield self.assertExported(items, list(header.values()), rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_based_on_item_classes(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1226,7 +1227,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_based_on_custom_filters(self): items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -1285,7 +1286,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_dicts(self): # When dicts are used, only keys from the first row are used as # a header for CSV, and all fields are used for JSON Lines. @@ -1298,7 +1299,7 @@ class TestFeedExport(TestFeedExportBase): yield self.assertExportedCsv(items, ["foo", "egg"], rows_csv) yield self.assertExportedJsonLines(items, rows_jl) - @defer.inlineCallbacks + @inlineCallbacks def test_export_tuple(self): items = [ {"foo": "bar1", "egg": "spam1"}, @@ -1309,7 +1310,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"foo": "bar1", "baz": ""}, {"foo": "bar2", "baz": "quux"}] yield self.assertExported(items, ["foo", "baz"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_feed_export_fields(self): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. @@ -1335,7 +1336,7 @@ class TestFeedExport(TestFeedExportBase): rows = [{"egg": "spam1", "baz": ""}, {"egg": "spam2", "baz": "quux2"}] yield self.assertExported(items, ["egg", "baz"], rows, settings=settings) - @defer.inlineCallbacks + @inlineCallbacks def test_export_encoding(self): items = [{"foo": "Test\xd6"}] @@ -1380,7 +1381,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_configs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1420,7 +1421,7 @@ class TestFeedExport(TestFeedExportBase): for fmt, expected in formats.items(): assert data[fmt] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_export_indentation(self): items = [ {"foo": ["bar"]}, @@ -1576,7 +1577,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[row["format"]] == row["expected"] - @defer.inlineCallbacks + @inlineCallbacks def test_init_exporters_storages_with_crawler(self): settings = { "FEED_EXPORTERS": {"csv": FromCrawlerCsvItemExporter}, @@ -1589,7 +1590,7 @@ class TestFeedExport(TestFeedExportBase): assert FromCrawlerCsvItemExporter.init_with_crawler assert FromCrawlerFileFeedStorage.init_with_crawler - @defer.inlineCallbacks + @inlineCallbacks def test_str_uri(self): settings = { "FEED_STORE_EMPTY": True, @@ -1598,7 +1599,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_no_data(settings) assert data["csv"] == b"" - @defer.inlineCallbacks + @inlineCallbacks def test_multiple_feeds_success_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1619,7 +1620,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Stored {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_multiple_feeds_failing_logs_blocking_feed_storage(self): settings = { "FEEDS": { @@ -1640,7 +1641,7 @@ class TestFeedExport(TestFeedExportBase): for fmt in ["json", "xml", "csv"]: assert f"Error storing {fmt} feed (2 items)" in str(log) - @defer.inlineCallbacks + @inlineCallbacks def test_extend_kwargs(self): items = [{"foo": "FOO", "bar": "BAR"}] @@ -1677,7 +1678,7 @@ class TestFeedExport(TestFeedExportBase): data = yield self.exported_data(items, settings) assert data[feed_options["format"]] == row["expected"] - @defer.inlineCallbacks + @inlineCallbacks def test_storage_file_no_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1699,7 +1700,7 @@ class TestFeedExport(TestFeedExportBase): yield self.exported_no_data(settings) assert Storage.open_file is Storage.store_file - @defer.inlineCallbacks + @inlineCallbacks def test_storage_file_postprocessing(self): @implementer(IFeedStorage) class Storage: @@ -1752,7 +1753,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): def _named_tempfile(self, name) -> str: return str(Path(self.temp_dir, name)) - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data with filename.""" @@ -1796,7 +1797,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data_stream.seek(0) return data_stream.read() - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin(self): filename = self._named_tempfile("gzip_file") @@ -1815,7 +1816,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid gzip data.") - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_0"): self.get_gzip_compressed( @@ -1852,7 +1853,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_mtime(self): filename_to_compressed = { self._named_tempfile("mtime_123"): self.get_gzip_compressed( @@ -1887,7 +1888,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_gzip_plugin_filename(self): filename_to_compressed = { self._named_tempfile("filename_FILE1"): self.get_gzip_compressed( @@ -1922,7 +1923,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin(self): filename = self._named_tempfile("lzma_file") @@ -1941,7 +1942,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except lzma.LZMAError: pytest.fail("Received invalid lzma data.") - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_format(self): filename_to_compressed = { self._named_tempfile("format_FORMAT_XZ"): lzma.compress( @@ -1974,7 +1975,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_check(self): filename_to_compressed = { self._named_tempfile("check_CHECK_NONE"): lzma.compress( @@ -2007,7 +2008,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_preset(self): filename_to_compressed = { self._named_tempfile("preset_PRESET_0"): lzma.compress( @@ -2040,11 +2041,11 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_lzma_plugin_filters(self): if "PyPy" in sys.version: # https://foss.heptapod.net/pypy/pypy/-/issues/3527 - raise unittest.SkipTest("lzma filters doesn't work in PyPy") + pytest.skip("lzma filters doesn't work in PyPy") filters = [{"id": lzma.FILTER_LZMA2}] compressed = lzma.compress(self.expected, filters=filters) @@ -2065,7 +2066,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = lzma.decompress(data[filename]) assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_bz2_plugin(self): filename = self._named_tempfile("bz2_file") @@ -2084,7 +2085,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): except OSError: pytest.fail("Received invalid bz2 data.") - @defer.inlineCallbacks + @inlineCallbacks def test_bz2_plugin_compresslevel(self): filename_to_compressed = { self._named_tempfile("compresslevel_1"): bz2.compress( @@ -2117,7 +2118,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert compressed == data[filename] assert result == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin(self): filename = self._named_tempfile("csv_file") @@ -2133,7 +2134,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = yield self.exported_data(self.items, settings) assert data[filename] == self.expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin_with_parameter(self): expected = b"foo\r\n\nbar\r\n\n" filename = self._named_tempfile("newline") @@ -2151,7 +2152,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): data = yield self.exported_data(self.items, settings) assert data[filename] == expected - @defer.inlineCallbacks + @inlineCallbacks def test_custom_plugin_with_compression(self): expected = b"foo\r\n\nbar\r\n\n" @@ -2196,7 +2197,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): result = decompressor(data[filename]) assert result == expected - @defer.inlineCallbacks + @inlineCallbacks def test_exports_compatibility_with_postproc(self): import marshal import pickle @@ -2254,7 +2255,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" - @defer.inlineCallbacks + @inlineCallbacks def run_and_export(self, spider_cls, settings): """Run spider with specified settings; return exported data.""" @@ -2276,7 +2277,7 @@ class TestBatchDeliveries(TestFeedExportBase): content[feed["format"]].append(file.read_bytes()) return content - @defer.inlineCallbacks + @inlineCallbacks def assertExportedJsonLines(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2298,7 +2299,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedCsv(self, items, header, rows, settings=None): settings = settings or {} settings.update( @@ -2318,7 +2319,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert list(got_batch) == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedXml(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2339,7 +2340,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMultiple(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2371,7 +2372,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, json_rows = json_rows[:batch_size], json_rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedPickle(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2393,7 +2394,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def assertExportedMarshal(self, items, rows, settings=None): settings = settings or {} settings.update( @@ -2415,7 +2416,7 @@ class TestBatchDeliveries(TestFeedExportBase): expected_batch, rows = rows[:batch_size], rows[batch_size:] assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ @@ -2444,7 +2445,7 @@ class TestBatchDeliveries(TestFeedExportBase): with pytest.raises(NotConfigured): FeedExporter(crawler) - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_not_store_empty(self): for fmt in ("json", "jsonlines", "xml", "csv"): settings = { @@ -2460,7 +2461,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert len(data[fmt]) == 0 - @defer.inlineCallbacks + @inlineCallbacks def test_export_no_items_store_empty(self): formats = ( ("json", b"[]"), @@ -2484,7 +2485,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = dict(data) assert data[fmt][0] == expctd - @defer.inlineCallbacks + @inlineCallbacks def test_export_multiple_configs(self): items = [ {"foo": "FOO", "bar": "BAR"}, @@ -2540,7 +2541,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_batch_item_count_feeds_setting(self): items = [{"foo": "FOO"}, {"foo": "FOO1"}] formats = { @@ -2564,7 +2565,7 @@ class TestBatchDeliveries(TestFeedExportBase): for expected_batch, got_batch in zip(expected, data[fmt]): assert got_batch == expected_batch - @defer.inlineCallbacks + @inlineCallbacks def test_batch_path_differ(self): """ Test that the name of all batch files differ from each other. @@ -2586,7 +2587,7 @@ class TestBatchDeliveries(TestFeedExportBase): data = yield self.exported_data(items, settings) assert len(items) == len(data["json"]) - @defer.inlineCallbacks + @inlineCallbacks def test_stats_batch_file_success(self): settings = { "FEEDS": { @@ -2604,7 +2605,7 @@ class TestBatchDeliveries(TestFeedExportBase): assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12 @pytest.mark.requires_boto3 - @defer.inlineCallbacks + @inlineCallbacks def test_s3_export(self): bucket = "mybucket" items = [ diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0605c2438..ef1806cc0 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -9,7 +9,7 @@ from ipaddress import IPv4Address from pathlib import Path from tempfile import mkdtemp from typing import TYPE_CHECKING -from unittest import mock, skipIf +from unittest import mock from urllib.parse import urlencode import pytest @@ -183,7 +183,7 @@ def get_client_certificate( return PrivateCertificate.loadPEM(pem) -@skipIf(not H2_ENABLED, "HTTP/2 support in Twisted is not enabled") +@pytest.mark.skipif(not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled") class TestHttps2ClientProtocol(TestCase): scheme = "https" key_file = Path(__file__).parent / "keys" / "localhost.key" diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 3c9f97631..047f8c610 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -2,7 +2,7 @@ import logging import pytest from testfixtures import LogCapture -from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure from twisted.trial.unittest import TestCase @@ -272,7 +272,7 @@ class TestShowOrSkipMessages(TestCase): }, } - @defer.inlineCallbacks + @inlineCallbacks def test_show_messages(self): crawler = get_crawler(ItemSpider, self.base_settings) with LogCapture() as lc: @@ -281,7 +281,7 @@ class TestShowOrSkipMessages(TestCase): assert "Crawled (200)