diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 0a5e61f7a..71f43365d 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -77,6 +77,7 @@ class Command(ScrapyCommand): # The crawler is created this way since the Shell manually handles the # crawling engine, so the set up in the crawl method won't work crawler = self.crawler_process._create_crawler(spidercls) + crawler._load_settings() # The Shell class needs a persistent engine in the crawler crawler.engine = crawler._create_engine() crawler.engine.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 49034c9f1..67e44541d 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -82,6 +82,7 @@ class Crawler: self._init_reactor: bool = init_reactor self.crawling: bool = False + self._settings_loaded: bool = False self._started: bool = False self.extensions: Optional[ExtensionManager] = None self.stats: Optional[StatsCollector] = None @@ -90,6 +91,53 @@ class Crawler: self.spider: Optional[Spider] = None self.engine: Optional[ExecutionEngine] = None + def _load_settings(self) -> None: + if self._settings_loaded: + return + self._settings_loaded = True + + self.addons.load_settings(self.settings) + self.stats = load_object(self.settings["STATS_CLASS"])(self) + + handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) + logging.root.addHandler(handler) + # lambda is assigned to Crawler attribute because this way it is not + # garbage collected after leaving the scope + self.__remove_handler = lambda: logging.root.removeHandler(handler) + self.signals.connect(self.__remove_handler, signals.engine_stopped) + + lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) + self.logformatter = lf_cls.from_crawler(self) + + self.request_fingerprinter = create_instance( + load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), + settings=self.settings, + crawler=self, + ) + + reactor_class: str = self.settings["TWISTED_REACTOR"] + event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] + if self._init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, event_loop) + else: + from twisted.internet import reactor # noqa: F401 + log_reactor_info() + if reactor_class: + verify_installed_reactor(reactor_class) + if is_asyncio_reactor_installed() and event_loop: + verify_installed_asyncio_event_loop(event_loop) + + self.extensions = ExtensionManager.from_crawler(self) + self.settings.freeze() + + d = dict(overridden_settings(self.settings)) + logger.info( + "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} + ) + @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: if self.crawling: @@ -104,49 +152,7 @@ class Crawler: try: self.spider = self._create_spider(*args, **kwargs) - - self.addons.load_settings(self.settings) - self.stats = load_object(self.settings["STATS_CLASS"])(self) - - handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL")) - logging.root.addHandler(handler) - # lambda is assigned to Crawler attribute because this way it is not - # garbage collected after leaving the scope - self.__remove_handler = lambda: logging.root.removeHandler(handler) - self.signals.connect(self.__remove_handler, signals.engine_stopped) - - lf_cls: Type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter = lf_cls.from_crawler(self) - - self.request_fingerprinter = create_instance( - load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), - settings=self.settings, - crawler=self, - ) - - reactor_class: str = self.settings["TWISTED_REACTOR"] - event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] - if self._init_reactor: - # this needs to be done after the spider settings are merged, - # but before something imports twisted.internet.reactor - if reactor_class: - install_reactor(reactor_class, event_loop) - else: - from twisted.internet import reactor # noqa: F401 - log_reactor_info() - if reactor_class: - verify_installed_reactor(reactor_class) - if is_asyncio_reactor_installed() and event_loop: - verify_installed_asyncio_event_loop(event_loop) - - self.extensions = ExtensionManager.from_crawler(self) - self.settings.freeze() - - d = dict(overridden_settings(self.settings)) - logger.info( - "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} - ) - + self._load_settings() self.engine = self._create_engine() start_requests = iter(self.spider.start_requests()) yield self.engine.open_spider(self.spider, start_requests) diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 44a30dc15..9397e78b9 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -69,28 +69,30 @@ def get_ftp_content_and_delete( return b"".join(ftp_data) +class TestSpider(Spider): + name = "test" + + def get_crawler( spidercls: Optional[Type[Spider]] = None, settings_dict: Optional[Dict[str, Any]] = None, prevent_warnings: bool = True, - disable_telnet: bool = True, ) -> Crawler: """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level priority. """ from scrapy.crawler import CrawlerRunner - from scrapy.spiders import Spider # Set by default settings that prevent deprecation warnings. settings: Dict[str, Any] = {} if prevent_warnings: settings["REQUEST_FINGERPRINTER_IMPLEMENTATION"] = "2.7" - if disable_telnet: - settings["TELNETCONSOLE_ENABLED"] = False settings.update(settings_dict or {}) runner = CrawlerRunner(settings) - return runner.create_crawler(spidercls or Spider) + crawler = runner.create_crawler(spidercls or TestSpider) + crawler._load_settings() + return crawler def get_pythonpath() -> str: diff --git a/tests/spiders.py b/tests/spiders.py index eeb0194eb..f29dea2a1 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -18,13 +18,6 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req -class NoRequestsSpider(Spider): - name = "no_request" - - def start_requests(self): - return [] - - class MockServerSpider(Spider): def __init__(self, mockserver=None, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/tests/test_addons.py b/tests/test_addons.py index c4b4b7ac9..aa1b760c2 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -1,15 +1,13 @@ import itertools +import unittest from typing import Any, Dict from unittest.mock import patch -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - +from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class SimpleAddon: @@ -53,17 +51,14 @@ class AddonTest(unittest.TestCase): class AddonManagerTest(unittest.TestCase): - @inlineCallbacks def test_load_settings(self): settings_dict = { "ADDONS": {"tests.test_addons.SimpleAddon": 0}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons self.assertIsInstance(manager.addons[0], SimpleAddon) - @inlineCallbacks def test_notconfigured(self): class NotConfiguredAddon: def update_settings(self, settings): @@ -72,12 +67,10 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {NotConfiguredAddon: 0}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons self.assertFalse(manager.addons) - @inlineCallbacks def test_load_settings_order(self): # Get three addons with different settings addonlist = [] @@ -89,25 +82,21 @@ class AddonManagerTest(unittest.TestCase): for ordered_addons in itertools.permutations(addonlist): expected_order = [a.number for a in ordered_addons] settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}} - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) manager = crawler.addons self.assertEqual([a.number for a in manager.addons], expected_order) self.assertEqual(crawler.settings.getint("KEY1"), expected_order[-1]) - @inlineCallbacks def test_create_instance(self): settings_dict = { "ADDONS": {"tests.test_addons.CreateInstanceAddon": 0}, "MYADDON": {"MYADDON_KEY": "val"}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) manager = crawler.addons self.assertIsInstance(manager.addons[0], CreateInstanceAddon) self.assertEqual(crawler.settings.get("MYADDON_KEY"), "val") - @inlineCallbacks def test_settings_priority(self): config = { "KEY": 15, # priority=addon @@ -115,15 +104,14 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {get_addon_cls(config): 1}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) self.assertEqual(crawler.settings.getint("KEY"), 15) settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) - crawler = runner.create_crawler(NoRequestsSpider) - yield crawler.crawl() + crawler = runner.create_crawler(Spider) + crawler._load_settings() self.assertEqual(crawler.settings.getint("KEY"), 15) settings_dict = { @@ -133,11 +121,9 @@ class AddonManagerTest(unittest.TestCase): settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") runner = CrawlerRunner(settings) - crawler = runner.create_crawler(NoRequestsSpider) - yield crawler.crawl() + crawler = runner.create_crawler(Spider) self.assertEqual(crawler.settings.getint("KEY"), 20) - @inlineCallbacks def test_fallback_workflow(self): FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -154,8 +140,7 @@ class AddonManagerTest(unittest.TestCase): settings_dict = { "ADDONS": {AddonWithFallback: 1}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) self.assertEqual( crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" ) @@ -168,14 +153,12 @@ class AddonManagerTest(unittest.TestCase): "ADDONS": {AddonWithFallback: 1}, "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) self.assertEqual( crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"], "AddonHandler" ) self.assertEqual(crawler.settings.get(FALLBACK_SETTING), "UserHandler") - @inlineCallbacks def test_logging_message(self): class LoggedAddon: def update_settings(self, settings): @@ -188,8 +171,7 @@ class AddonManagerTest(unittest.TestCase): } addon = LoggedAddon() create_instance_mock.return_value = addon - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings_dict) logger_mock.info.assert_called_once_with( "Enabled addons:\n%(addons)s", {"addons": [addon]}, diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 72d06deab..6589381f3 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -7,8 +7,6 @@ from scrapy.utils.testproc import ProcessTest from scrapy.utils.testsite import SiteTest from tests import NON_EXISTING_RESOLVABLE, tests_datadir -raise unittest.SkipTest("Broken for now") - class ShellTest(ProcessTest, SiteTest, unittest.TestCase): command = "shell" diff --git a/tests/test_commands.py b/tests/test_commands.py index 05be33c73..b1d7be628 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -26,8 +26,7 @@ from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from tests.spiders import NoRequestsSpider -from tests.test_crawler import ExceptionSpider +from tests.test_crawler import ExceptionSpider, NoRequestsSpider class CommandSettings(unittest.TestCase): @@ -713,7 +712,7 @@ class BadSpider(scrapy.Spider): def test_run_good_spider(self): proc, _, _ = self.runspider( - "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider) + "import scrapy\n" + inspect.getsource(NoRequestsSpider) ) ret = proc.returncode self.assertEqual(ret, 0) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f962cecc8..120991ae7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -10,7 +10,6 @@ import pytest from packaging.version import parse as parse_version from pytest import mark, raises from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib import __version__ as w3lib_version @@ -24,7 +23,6 @@ from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer, get_mockserver_env -from tests.spiders import NoRequestsSpider class BaseCrawlerTest(unittest.TestCase): @@ -72,7 +70,6 @@ class CrawlerLoggingTestCase(unittest.TestCase): get_crawler(MySpider) assert get_scrapy_root_handler() is None - @inlineCallbacks def test_spider_custom_settings_log_level(self): log_file = Path(self.mktemp()) log_file.write_text("previous message\n", encoding="utf-8") @@ -82,20 +79,20 @@ class CrawlerLoggingTestCase(unittest.TestCase): custom_settings = { "LOG_LEVEL": "INFO", "LOG_FILE": str(log_file), + # settings to avoid extra warnings + "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", + "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE, } - def start_requests(self): - logging.debug("debug message") - logging.info("info message") - logging.warning("warning message") - logging.error("error message") - return [] - configure_logging() self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG) crawler = get_crawler(MySpider) - yield crawler.crawl() self.assertEqual(get_scrapy_root_handler().level, logging.INFO) + info_count = crawler.stats.get_value("log_count/INFO") + logging.debug("debug message") + logging.info("info message") + logging.warning("warning message") + logging.error("error message") logged = log_file.read_text(encoding="utf-8") @@ -106,7 +103,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): self.assertIn("error message", logged) self.assertEqual(crawler.stats.get_value("log_count/ERROR"), 1) self.assertEqual(crawler.stats.get_value("log_count/WARNING"), 1) - # self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) TODO + self.assertEqual(crawler.stats.get_value("log_count/INFO") - info_count, 1) self.assertEqual(crawler.stats.get_value("log_count/DEBUG", 0), 0) def test_spider_custom_settings_log_append(self): @@ -183,6 +180,13 @@ class ExceptionSpider(scrapy.Spider): raise ValueError("Exception in from_crawler method") +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + def start_requests(self): + return [] + + @mark.usefixtures("reactor_pytest") class CrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index d64651211..062e8a8b4 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -3,33 +3,33 @@ from unittest import mock from pytest import mark from twisted.internet import defer -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred from twisted.python.failure import Failure from twisted.trial.unittest import TestCase from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue -from tests.spiders import NoRequestsSpider class ManagerTestCase(TestCase): settings_dict = None - @inlineCallbacks def setUp(self): - self.crawler = get_crawler(NoRequestsSpider, self.settings_dict) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider, self.settings_dict) + self.spider = self.crawler._create_spider("foo") self.mwman = DownloaderMiddlewareManager.from_crawler(self.crawler) - yield self.mwman.open_spider(self.spider) + # some mw depends on stats collector + self.crawler.stats.open_spider(self.spider) + return self.mwman.open_spider(self.spider) def tearDown(self): + self.crawler.stats.close_spider(self.spider, "") return self.mwman.close_spider(self.spider) - @inlineCallbacks def _download(self, request, response=None): """Executes downloader mw manager's download method and returns the result (Request or Response) or raise exception in case of @@ -41,21 +41,26 @@ class ManagerTestCase(TestCase): def download_func(**kwargs): return response - ret = yield self.mwman.download(download_func, request, self.spider) + dfd = self.mwman.download(download_func, request, self.spider) + # catch deferred result and return the value + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + ret = results[0] + if isinstance(ret, Failure): + ret.raiseException() return ret class DefaultsTest(ManagerTestCase): """Tests default behavior with default settings""" - @inlineCallbacks def test_request_response(self): req = Request("http://example.com/index.html") resp = Response(req.url, status=200) - ret = yield self._download(req, resp) + ret = self._download(req, resp) self.assertTrue(isinstance(ret, Response), "Non-response returned") - @inlineCallbacks def test_3xx_and_invalid_gzipped_body_must_redirect(self): """Regression test for a failure when redirecting a compressed request. @@ -81,7 +86,7 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - ret = yield self._download(request=req, response=resp) + ret = self._download(request=req, response=resp) self.assertTrue(isinstance(ret, Request), f"Not redirected: {ret!r}") self.assertEqual( to_bytes(ret.url), @@ -89,7 +94,6 @@ class DefaultsTest(ManagerTestCase): "Not redirected to location header", ) - @inlineCallbacks def test_200_and_invalid_gzipped_body_must_fail(self): req = Request("http://example.com") body = b"

You are being redirected

" @@ -104,14 +108,12 @@ class DefaultsTest(ManagerTestCase): "Location": "http://example.com/login", }, ) - with self.assertRaises(OSError): - yield self._download(request=req, response=resp) + self.assertRaises(OSError, self._download, request=req, response=resp) class ResponseFromProcessRequestTest(ManagerTestCase): """Tests middleware returning a response from process_request.""" - @inlineCallbacks def test_download_func_not_called(self): resp = Response("http://example.com/index.html") @@ -123,8 +125,12 @@ class ResponseFromProcessRequestTest(ManagerTestCase): req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) @@ -191,7 +197,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): class MiddlewareUsingDeferreds(ManagerTestCase): """Middlewares using Deferreds should work""" - @inlineCallbacks def test_deferred(self): resp = Response("http://example.com/index.html") @@ -208,8 +213,12 @@ class MiddlewareUsingDeferreds(ManagerTestCase): self.mwman._add_middleware(DeferredMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) @@ -217,7 +226,6 @@ class MiddlewareUsingDeferreds(ManagerTestCase): class MiddlewareUsingCoro(ManagerTestCase): """Middlewares using asyncio coroutines should work""" - @inlineCallbacks def test_asyncdef(self): resp = Response("http://example.com/index.html") @@ -229,12 +237,15 @@ class MiddlewareUsingCoro(ManagerTestCase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) @mark.only_asyncio() - @inlineCallbacks def test_asyncdef_asyncio(self): resp = Response("http://example.com/index.html") @@ -247,6 +258,10 @@ class MiddlewareUsingCoro(ManagerTestCase): self.mwman._add_middleware(CoroMiddleware()) req = Request("http://example.com/index.html") download_func = mock.MagicMock() - result = yield self.mwman.download(download_func, req, self.spider) - self.assertIs(result, resp) + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + + self.assertIs(results[0], resp) self.assertFalse(download_func.called) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index a8a687da7..f80eff3e6 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -2,31 +2,27 @@ import email.utils import shutil import tempfile import time +import unittest from contextlib import contextmanager -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.http import HtmlResponse, Request, Response from scrapy.settings import Settings +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class _BaseTest(unittest.TestCase): storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" policy_class = "scrapy.extensions.httpcache.RFC2616Policy" - @inlineCallbacks def setUp(self): self.yesterday = email.utils.formatdate(time.time() - 86400) self.today = email.utils.formatdate() self.tomorrow = email.utils.formatdate(time.time() + 86400) - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("example.com") self.tmpdir = tempfile.mkdtemp() self.request = Request("http://www.example.com", headers={"User-Agent": "test"}) self.response = Response( @@ -35,6 +31,7 @@ class _BaseTest(unittest.TestCase): body=b"test body", status=202, ) + self.crawler.stats.open_spider(self.spider) def tearDown(self): self.crawler.stats.close_spider(self.spider, "") @@ -569,3 +566,7 @@ class RFC2616PolicyTest(DefaultStorageTest): res2 = self._process_requestresponse(mw, req0, None) self.assertEqualResponse(res1, res2) assert "cached" in res2.flags + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 4a579c061..9dad056de 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,10 +1,9 @@ from gzip import GzipFile from io import BytesIO from pathlib import Path +from unittest import SkipTest, TestCase from warnings import catch_warnings -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from w3lib.encoding import resolve_encoding from scrapy.downloadermiddlewares.httpcompression import ( @@ -14,10 +13,10 @@ from scrapy.downloadermiddlewares.httpcompression import ( from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes +from scrapy.spiders import Spider from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir -from tests.spiders import NoRequestsSpider SAMPLEDIR = Path(tests_datadir, "compressed") @@ -39,13 +38,12 @@ FORMAT = { } -class HttpCompressionTest(unittest.TestCase): - @inlineCallbacks +class HttpCompressionTest(TestCase): def setUp(self): - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("scrapytest.org") self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -133,7 +131,7 @@ class HttpCompressionTest(unittest.TestCase): try: import brotli # noqa: F401 except ImportError: - raise unittest.SkipTest("no brotli") + raise SkipTest("no brotli") response = self._getresponse("br") request = response.request self.assertEqual(response.headers["Content-Encoding"], b"br") @@ -148,7 +146,7 @@ class HttpCompressionTest(unittest.TestCase): try: import zstandard # noqa: F401 except ImportError: - raise unittest.SkipTest("no zstd support (zstandard)") + raise SkipTest("no zstd support (zstandard)") raw_content = None for check_key in FORMAT: if not check_key.startswith("zstd-"): @@ -376,15 +374,13 @@ class HttpCompressionTest(unittest.TestCase): self.assertStatsEqual("httpcompression/response_bytes", None) -class HttpCompressionSubclassTest(unittest.TestCase): - @inlineCallbacks +class HttpCompressionSubclassTest(TestCase): def test_init_missing_stats(self): class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): def __init__(self): super().__init__() - crawler = get_crawler(NoRequestsSpider) - yield crawler.crawl() + crawler = get_crawler(Spider) with catch_warnings(record=True) as caught_warnings: HttpCompressionMiddlewareSubclass.from_crawler(crawler) messages = tuple( diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index f94958ff7..97ae1e29a 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -1,9 +1,9 @@ import logging +import unittest import warnings from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.internet.error import ( ConnectError, ConnectionDone, @@ -11,7 +11,6 @@ from twisted.internet.error import ( DNSLookupError, TCPTimedOutError, ) -from twisted.trial import unittest from twisted.web.client import ResponseFailed from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request @@ -20,15 +19,12 @@ from scrapy.http import Request, Response from scrapy.settings.default_settings import RETRY_EXCEPTIONS from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class RetryTest(unittest.TestCase): - @inlineCallbacks def setUp(self): - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("foo") self.mw = RetryMiddleware.from_crawler(self.crawler) self.mw.max_retry_times = 2 @@ -164,19 +160,16 @@ class RetryTest(unittest.TestCase): class MaxRetryTimesTest(unittest.TestCase): invalid_url = "http://www.scrapytest.org/invalid_url" - @inlineCallbacks def get_spider_and_middleware(self, settings=None): - crawler = get_crawler(NoRequestsSpider, settings or {}) - yield crawler.crawl() - spider = crawler.spider + crawler = get_crawler(Spider, settings or {}) + spider = crawler._create_spider("foo") middleware = RetryMiddleware.from_crawler(crawler) return spider, middleware - @inlineCallbacks def test_with_settings_zero(self): max_retry_times = 0 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, @@ -186,10 +179,9 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_metakey_zero(self): max_retry_times = 0 - spider, middleware = yield self.get_spider_and_middleware() + spider, middleware = self.get_spider_and_middleware() meta = {"max_retry_times": max_retry_times} req = Request(self.invalid_url, meta=meta) self._test_retry( @@ -200,11 +192,10 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_without_metakey(self): max_retry_times = 5 settings = {"RETRY_TIMES": max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) req = Request(self.invalid_url) self._test_retry( req, @@ -214,7 +205,6 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_metakey_greater(self): meta_max_retry_times = 3 middleware_max_retry_times = 2 @@ -223,7 +213,7 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) self._test_retry( req1, @@ -240,7 +230,6 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_metakey_lesser(self): meta_max_retry_times = 4 middleware_max_retry_times = 5 @@ -249,7 +238,7 @@ class MaxRetryTimesTest(unittest.TestCase): req2 = Request(self.invalid_url) settings = {"RETRY_TIMES": middleware_max_retry_times} - spider, middleware = yield self.get_spider_and_middleware(settings) + spider, middleware = self.get_spider_and_middleware(settings) self._test_retry( req1, @@ -266,10 +255,9 @@ class MaxRetryTimesTest(unittest.TestCase): middleware=middleware, ) - @inlineCallbacks def test_with_dont_retry(self): max_retry_times = 4 - spider, middleware = yield self.get_spider_and_middleware() + spider, middleware = self.get_spider_and_middleware() meta = { "max_retry_times": max_retry_times, "dont_retry": True, @@ -304,16 +292,13 @@ class MaxRetryTimesTest(unittest.TestCase): class GetRetryRequestTest(unittest.TestCase): - @inlineCallbacks def get_spider(self, settings=None): - crawler = get_crawler(NoRequestsSpider, settings or {}) - yield crawler.crawl() - return crawler.spider + crawler = get_crawler(Spider, settings or {}) + return crawler._create_spider("foo") - @inlineCallbacks def test_basic_usage(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture() as log: new_request = get_retry_request( request, @@ -337,10 +322,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_max_retries_reached(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() max_retry_times = 0 with LogCapture() as log: new_request = get_retry_request( @@ -361,10 +345,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_one_retry(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture() as log: new_request = get_retry_request( request, @@ -389,9 +372,8 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_two_retries(self): - spider = yield self.get_spider() + spider = self.get_spider() request = Request("https://example.com") new_request = request max_retry_times = 2 @@ -445,10 +427,9 @@ class GetRetryRequestTest(unittest.TestCase): with self.assertRaises(TypeError): get_retry_request(request) # pylint: disable=missing-kwoa - @inlineCallbacks def test_max_retry_times_setting(self): max_retry_times = 0 - spider = yield self.get_spider({"RETRY_TIMES": max_retry_times}) + spider = self.get_spider({"RETRY_TIMES": max_retry_times}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -456,10 +437,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) - @inlineCallbacks def test_max_retry_times_meta(self): max_retry_times = 0 - spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) meta = {"max_retry_times": max_retry_times} request = Request("https://example.com", meta=meta) new_request = get_retry_request( @@ -468,10 +448,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) - @inlineCallbacks def test_max_retry_times_argument(self): max_retry_times = 0 - spider = yield self.get_spider({"RETRY_TIMES": max_retry_times + 1}) + spider = self.get_spider({"RETRY_TIMES": max_retry_times + 1}) meta = {"max_retry_times": max_retry_times + 1} request = Request("https://example.com", meta=meta) new_request = get_retry_request( @@ -481,10 +460,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request, None) - @inlineCallbacks def test_priority_adjust_setting(self): priority_adjust = 1 - spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust}) + spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -492,10 +470,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request.priority, priority_adjust) - @inlineCallbacks def test_priority_adjust_argument(self): priority_adjust = 1 - spider = yield self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1}) + spider = self.get_spider({"RETRY_PRIORITY_ADJUST": priority_adjust + 1}) request = Request("https://example.com") new_request = get_retry_request( request, @@ -504,10 +481,9 @@ class GetRetryRequestTest(unittest.TestCase): ) self.assertEqual(new_request.priority, priority_adjust) - @inlineCallbacks def test_log_extra_retry_success(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture(attributes=("spider",)) as log: get_retry_request( request, @@ -515,10 +491,9 @@ class GetRetryRequestTest(unittest.TestCase): ) log.check_present(spider) - @inlineCallbacks def test_log_extra_retries_exceeded(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() with LogCapture(attributes=("spider",)) as log: get_retry_request( request, @@ -527,10 +502,9 @@ class GetRetryRequestTest(unittest.TestCase): ) log.check_present(spider) - @inlineCallbacks def test_reason_string(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = "because" with LogCapture() as log: get_retry_request( @@ -550,10 +524,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_builtin_exception(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = NotImplementedError() expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: @@ -576,10 +549,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_builtin_exception_class(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = NotImplementedError expected_reason_string = "builtins.NotImplementedError" with LogCapture() as log: @@ -602,10 +574,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_custom_exception(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = IgnoreRequest() expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: @@ -628,10 +599,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_reason_custom_exception_class(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = IgnoreRequest expected_reason_string = "scrapy.exceptions.IgnoreRequest" with LogCapture() as log: @@ -654,11 +624,10 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_custom_logger(self): logger = logging.getLogger("custom-logger") request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = "because" with LogCapture() as log: get_retry_request( @@ -675,10 +644,9 @@ class GetRetryRequestTest(unittest.TestCase): ) ) - @inlineCallbacks def test_custom_stats_key(self): request = Request("https://example.com") - spider = yield self.get_spider() + spider = self.get_spider() expected_reason = "because" stats_key = "custom_retry" get_retry_request( @@ -692,3 +660,7 @@ class GetRetryRequestTest(unittest.TestCase): f"{stats_key}/reason_count/{expected_reason}", ): self.assertEqual(spider.crawler.stats.get_value(stat), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py index 55ae0c2b7..39dfe9ab5 100644 --- a/tests/test_downloadermiddleware_stats.py +++ b/tests/test_downloadermiddleware_stats.py @@ -1,29 +1,27 @@ import warnings from itertools import product - -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest +from unittest import TestCase from scrapy.downloadermiddlewares.stats import DownloaderStats from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response +from scrapy.spiders import Spider from scrapy.utils.response import response_httprepr from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class MyException(Exception): pass -class TestDownloaderStats(unittest.TestCase): - @inlineCallbacks +class TestDownloaderStats(TestCase): def setUp(self): - self.crawler = get_crawler(NoRequestsSpider) - yield self.crawler.crawl() - self.spider = self.crawler.spider + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider("scrapytest.org") self.mw = DownloaderStats(self.crawler.stats) + self.crawler.stats.open_spider(self.spider) + self.req = Request("http://scrapytest.org") self.res = Response("scrapytest.org", status=400) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 1ba6125b2..aa0975555 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -2,25 +2,22 @@ import hashlib import shutil import sys import tempfile +import unittest from pathlib import Path from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from scrapy.core.scheduler import Scheduler from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider +from tests.spiders import SimpleSpider -@inlineCallbacks def _get_dupefilter(*, crawler=None, settings=None, open=True): if crawler is None: - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) dupefilter = scheduler.df if open: @@ -47,51 +44,41 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): class DirectDupeFilter: method = "n/a" - def open(self): - pass - class RFPDupeFilterTest(unittest.TestCase): - @inlineCallbacks def test_df_from_crawler_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, "from_crawler") - @inlineCallbacks def test_df_from_settings_scheduler(self): settings = { "DUPEFILTER_DEBUG": True, "DUPEFILTER_CLASS": FromSettingsRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, "from_settings") - @inlineCallbacks def test_df_direct_scheduler(self): settings = { "DUPEFILTER_CLASS": DirectDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertEqual(scheduler.df.method, "n/a") - @inlineCallbacks def test_filter(self): - dupefilter = yield _get_dupefilter() + dupefilter = _get_dupefilter() r1 = Request("http://scrapytest.org/1") r2 = Request("http://scrapytest.org/2") r3 = Request("http://scrapytest.org/2") @@ -104,14 +91,13 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") - @inlineCallbacks def test_dupefilter_path(self): r1 = Request("http://scrapytest.org/1") r2 = Request("http://scrapytest.org/2") path = tempfile.mkdtemp() try: - df = yield _get_dupefilter(settings={"JOBDIR": path}, open=False) + df = _get_dupefilter(settings={"JOBDIR": path}, open=False) try: df.open() assert not df.request_seen(r1) @@ -119,7 +105,7 @@ class RFPDupeFilterTest(unittest.TestCase): finally: df.close("finished") - df2 = yield _get_dupefilter(settings={"JOBDIR": path}, open=False) + df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False) assert df != df2 try: df2.open() @@ -131,13 +117,12 @@ class RFPDupeFilterTest(unittest.TestCase): finally: shutil.rmtree(path) - @inlineCallbacks def test_request_fingerprint(self): """Test if customization of request_fingerprint method will change output of request_seen. """ - dupefilter = yield _get_dupefilter() + dupefilter = _get_dupefilter() r1 = Request("http://scrapytest.org/index.html") r2 = Request("http://scrapytest.org/INDEX.html") @@ -153,14 +138,13 @@ class RFPDupeFilterTest(unittest.TestCase): return fp.digest() settings = {"REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter} - case_insensitive_dupefilter = yield _get_dupefilter(settings=settings) + case_insensitive_dupefilter = _get_dupefilter(settings=settings) assert not case_insensitive_dupefilter.request_seen(r1) assert case_insensitive_dupefilter.request_seen(r2) case_insensitive_dupefilter.close("finished") - @inlineCallbacks def test_seenreq_newlines(self): """Checks against adding duplicate \r to line endings on Windows platforms.""" @@ -168,8 +152,7 @@ class RFPDupeFilterTest(unittest.TestCase): r1 = Request("http://scrapytest.org/1") path = tempfile.mkdtemp() - crawler = get_crawler(NoRequestsSpider, settings_dict={"JOBDIR": path}) - yield crawler.crawl() + crawler = get_crawler(settings_dict={"JOBDIR": path}) try: scheduler = Scheduler.from_crawler(crawler) df = scheduler.df @@ -178,10 +161,7 @@ class RFPDupeFilterTest(unittest.TestCase): df.close("finished") with Path(path, "requests.seen").open("rb") as seen_file: - try: - line = next(seen_file).decode() - except StopIteration: - return + line = next(seen_file).decode() assert not line.endswith("\r\r\n") if sys.platform == "win32": assert line.endswith("\r\n") @@ -191,7 +171,6 @@ class RFPDupeFilterTest(unittest.TestCase): finally: shutil.rmtree(path) - @inlineCallbacks def test_log(self): with LogCapture() as log: settings = { @@ -199,10 +178,9 @@ class RFPDupeFilterTest(unittest.TestCase): "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() - spider = crawler.spider - dupefilter = yield _get_dupefilter(crawler=crawler) + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request("http://scrapytest.org/index.html") @@ -222,7 +200,6 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") - @inlineCallbacks def test_log_debug(self): with LogCapture() as log: settings = { @@ -230,10 +207,9 @@ class RFPDupeFilterTest(unittest.TestCase): "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() - spider = crawler.spider - dupefilter = yield _get_dupefilter(crawler=crawler) + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request( @@ -263,17 +239,15 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close("finished") - @inlineCallbacks def test_log_debug_default_dupefilter(self): with LogCapture() as log: settings = { "DUPEFILTER_DEBUG": True, "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() - spider = crawler.spider - dupefilter = yield _get_dupefilter(crawler=crawler) + crawler = get_crawler(SimpleSpider, settings_dict=settings) + spider = SimpleSpider.from_crawler(crawler) + dupefilter = _get_dupefilter(crawler=crawler) r1 = Request("http://scrapytest.org/index.html") r2 = Request( diff --git a/tests/test_engine.py b/tests/test_engine.py index 5deae5146..8d7afb6a1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -420,16 +420,12 @@ class EngineTest(unittest.TestCase): @defer.inlineCallbacks def test_close_downloader(self): - crawler = get_crawler(TestSpider) - yield crawler.crawl() - e = ExecutionEngine(crawler, lambda _: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.close() @defer.inlineCallbacks def test_start_already_running_exception(self): - crawler = get_crawler(TestSpider) - yield crawler.crawl() - e = ExecutionEngine(crawler, lambda _: None) + e = ExecutionEngine(get_crawler(TestSpider), lambda _: None) yield e.open_spider(TestSpider(), []) e.start() try: diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ce8452b5f..6e5fb0325 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,8 +1,6 @@ import datetime import typing - -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest +import unittest from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog @@ -61,44 +59,33 @@ class TestExtPeriodicLog(PeriodicLog): self.stats._stats = stats_dump_2 -@inlineCallbacks def extension(settings=None): - crawler = Crawler( - MetaSpider, - settings=settings, - ) - yield crawler.crawl() + crawler = Crawler(MetaSpider, settings=settings) + crawler._load_settings() return TestExtPeriodicLog.from_crawler(crawler) class TestPeriodicLog(unittest.TestCase): - @inlineCallbacks def test_extension_enabled(self): # Expected that settings for this extension loaded succesfully # And on certain conditions - extension raising NotConfigured # "PERIODIC_LOG_STATS": True -> set to {"enabled": True} # due to TypeError exception from settings.getdict - assert (yield extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60})) + assert extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 60}) # "PERIODIC_LOG_STATS": "True" -> set to {"enabled": True} # due to JSONDecodeError(ValueError) exception from settings.getdict - assert ( - yield extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) - ) + assert extension({"PERIODIC_LOG_STATS": "True", "LOGSTATS_INTERVAL": 60}) # The ame for PERIODIC_LOG_DELTA: - assert (yield extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})) - assert ( - yield extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) - ) + assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) + assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) - @inlineCallbacks def test_log_delta(self): - @inlineCallbacks def emulate(settings=None): spider = MetaSpider() - ext = yield extension(settings) + ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_delta() @@ -107,9 +94,8 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - @inlineCallbacks def check(settings: dict, condition: typing.Callable): - ext, a, b = yield emulate(settings) + ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) ] @@ -118,49 +104,45 @@ class TestPeriodicLog(unittest.TestCase): ] # Including all - yield check( - {"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float)) - ) + check({"PERIODIC_LOG_DELTA": True}, lambda k, v: isinstance(v, (int, float))) # include: - yield check( + check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"]}}, lambda k, v: isinstance(v, (int, float)) and "downloader/" in k, ) # include multiple - yield check( + check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" in k or "scheduler/" in k), ) # exclude - yield check( + check( {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/"]}}, lambda k, v: isinstance(v, (int, float)) and "downloader/" not in k, ) # exclude multiple - yield check( + check( {"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" not in k and "scheduler/" not in k), ) # include exclude combined - yield check( + check( {"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: isinstance(v, (int, float)) and ("downloader/" in k and "bytes" not in k), ) - @inlineCallbacks def test_log_stats(self): - @inlineCallbacks def emulate(settings=None): spider = MetaSpider() - ext = yield extension(settings) + ext = extension(settings) ext.spider_opened(spider) ext.set_a() a = ext.log_crawler_stats() @@ -169,9 +151,8 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - @inlineCallbacks def check(settings: dict, condition: typing.Callable): - ext, a, b = yield emulate(settings) + ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) ] @@ -180,34 +161,35 @@ class TestPeriodicLog(unittest.TestCase): ] # Including all - yield check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) + check({"PERIODIC_LOG_STATS": True}, lambda k, v: True) # include: - yield check( + check( {"PERIODIC_LOG_STATS": {"include": ["downloader/"]}}, lambda k, v: "downloader/" in k, ) # include multiple - yield check( + check( {"PERIODIC_LOG_STATS": {"include": ["downloader/", "scheduler/"]}}, lambda k, v: "downloader/" in k or "scheduler/" in k, ) # exclude - yield check( + check( {"PERIODIC_LOG_STATS": {"exclude": ["downloader/"]}}, lambda k, v: "downloader/" not in k, ) # exclude multiple - yield check( + check( {"PERIODIC_LOG_STATS": {"exclude": ["downloader/", "scheduler/"]}}, lambda k, v: "downloader/" not in k and "scheduler/" not in k, ) # include exclude combined - yield check( + check( {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}}, lambda k, v: "downloader/" in k and "bytes" not in k, ) + # diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 6f35510b7..9fd680e9f 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -9,7 +9,7 @@ from scrapy.utils.test import get_crawler class TelnetExtensionTest(unittest.TestCase): def _get_console_and_portal(self, settings=None): - crawler = get_crawler(settings_dict=settings, disable_telnet=False) + crawler = get_crawler(settings_dict=settings) console = TelnetConsole(crawler) # This function has some side effects we don't need for this test diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 875abdb1f..6b82974fa 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -26,7 +26,6 @@ import lxml.etree import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from w3lib.url import file_uri_to_path, path_to_file_uri from zope.interface import implementer @@ -52,7 +51,7 @@ from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no_boto from tests.mockserver import MockFTPServer, MockServer -from tests.spiders import ItemSpider, NoRequestsSpider +from tests.spiders import ItemSpider def path_to_url(path): @@ -2748,14 +2747,13 @@ class FeedExporterSignalsTest(unittest.TestCase): d.callback(None) return d - @inlineCallbacks def run_signaled_feed_exporter( self, feed_exporter_signal_handler, feed_slot_signal_handler ): - crawler = get_crawler(NoRequestsSpider, settings_dict=self.settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=self.settings) feed_exporter = FeedExporter.from_crawler(crawler) - spider = crawler.spider + spider = scrapy.Spider("default") + spider.crawler = crawler crawler.signals.connect( feed_exporter_signal_handler, signal=signals.feed_exporter_closed, @@ -2766,26 +2764,24 @@ class FeedExporterSignalsTest(unittest.TestCase): feed_exporter.open_spider(spider) for item in self.items: feed_exporter.item_scraped(item, spider) - yield defer.ensureDeferred(feed_exporter.close_spider(spider)) + defer.ensureDeferred(feed_exporter.close_spider(spider)) - @inlineCallbacks def test_feed_exporter_signals_sent(self): self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - yield self.run_signaled_feed_exporter( + self.run_signaled_feed_exporter( self.feed_exporter_closed_signal_handler, self.feed_slot_closed_signal_handler, ) self.assertTrue(self.feed_slot_closed_received) self.assertTrue(self.feed_exporter_closed_received) - @inlineCallbacks def test_feed_exporter_signals_sent_deferred(self): self.feed_exporter_closed_received = False self.feed_slot_closed_received = False - yield self.run_signaled_feed_exporter( + self.run_signaled_feed_exporter( self.feed_exporter_closed_signal_handler_deferred, self.feed_slot_closed_signal_handler_deferred, ) diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 0d769f29b..bf96f17b6 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -15,7 +15,6 @@ import attr import pytest from itemadapter import ItemAdapter from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.trial import unittest from scrapy.http import Request, Response @@ -36,18 +35,15 @@ from scrapy.utils.test import ( skip_if_no_boto, ) from tests.mockserver import MockFTPServer -from tests.spiders import NoRequestsSpider from .test_pipeline_media import _mocked_download_func class FilesPipelineTestCase(unittest.TestCase): - @inlineCallbacks def setUp(self): self.tempdir = mkdtemp() settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func self.pipeline.open_spider(None) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 8a13ea552..d655eb128 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -14,11 +14,11 @@ from scrapy.pipelines.files import FileException from scrapy.pipelines.images import ImagesPipeline from scrapy.pipelines.media import MediaPipeline from scrapy.settings import Settings +from scrapy.spiders import Spider from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider try: from PIL import Image # noqa: imported just to check for the import error @@ -40,11 +40,10 @@ class BaseMediaPipelineTestCase(unittest.TestCase): pipeline_class = MediaPipeline settings = None - @inlineCallbacks def setUp(self): - crawler = get_crawler(NoRequestsSpider, self.settings) - yield crawler.crawl() - self.spider = crawler.spider + spider_cls = Spider + self.spider = spider_cls("media.com") + crawler = get_crawler(spider_cls, self.settings) self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.download_func = _mocked_download_func self.pipe.open_spider(self.spider) @@ -431,14 +430,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline): class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase): skip = skip_pillow - @inlineCallbacks def setUp(self): settings_dict = { "IMAGES_STORE": "store-uri", "IMAGES_THUMBS": {"small": (50, 50)}, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings_dict) - yield crawler.crawl() + crawler = get_crawler(spidercls=None, settings_dict=settings_dict) self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler) self.pipe.download_func = _mocked_download_func self.pipe.open_spider(None) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 54c77eb68..ef9b360c4 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,10 +1,10 @@ import collections import shutil import tempfile +import unittest from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest +from twisted.trial.unittest import TestCase from scrapy.core.downloader import Downloader from scrapy.core.scheduler import Scheduler @@ -15,7 +15,6 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import NoRequestsSpider MockEngine = collections.namedtuple("MockEngine", ["downloader"]) MockSlot = collections.namedtuple("MockSlot", ["active"]) @@ -310,7 +309,7 @@ class StartUrlsSpider(Spider): pass -class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase): +class TestIntegrationWithDownloaderAwareInMemory(TestCase): def setUp(self): self.crawler = get_crawler( spidercls=StartUrlsSpider, @@ -337,19 +336,16 @@ class TestIntegrationWithDownloaderAwareInMemory(unittest.TestCase): class TestIncompatibility(unittest.TestCase): - @inlineCallbacks def _incompatible(self): settings = dict( SCHEDULER_PRIORITY_QUEUE="scrapy.pqueues.DownloaderAwarePriorityQueue", CONCURRENT_REQUESTS_PER_IP=1, ) - crawler = get_crawler(NoRequestsSpider, settings) - yield crawler.crawl() - spider = crawler.spider + crawler = get_crawler(Spider, settings) scheduler = Scheduler.from_crawler(crawler) + spider = Spider(name="spider") scheduler.open(spider) - @inlineCallbacks def test_incompatibility(self): with self.assertRaises(ValueError): - yield self._incompatible() + self._incompatible() diff --git a/tests/test_spider.py b/tests/test_spider.py index a88d9b505..00da3d485 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -114,7 +114,7 @@ class SpiderTest(unittest.TestCase): spider.settings.set("TEST1", "spider_instance", priority="spider") return spider - crawler = get_crawler(TestSpider, settings_dict=project_settings) + crawler = Crawler(TestSpider, project_settings) self.assertEqual(crawler.settings.get("TEST1"), "spider") self.assertEqual(crawler.settings.get("TEST2"), "spider") self.assertEqual(crawler.settings.get("TEST3"), "project") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 2fcb5b364..1d5a887cc 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,9 +1,9 @@ import logging +from unittest import TestCase from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks -from twisted.trial.unittest import TestCase +from twisted.trial.unittest import TestCase as TrialTestCase from scrapy.http import Request, Response from scrapy.settings import Settings @@ -11,7 +11,7 @@ from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import MockServerSpider, NoRequestsSpider +from tests.spiders import MockServerSpider class _HttpErrorSpider(MockServerSpider): @@ -59,11 +59,9 @@ def _responses(request, status_codes): class TestHttpErrorMiddleware(TestCase): - @inlineCallbacks def setUp(self): - crawler = get_crawler(NoRequestsSpider) - yield crawler.crawl() - self.spider = crawler.spider + crawler = get_crawler(Spider) + self.spider = Spider.from_crawler(crawler, name="foo") self.mw = HttpErrorMiddleware(Settings({})) self.req = Request("http://scrapytest.org") self.res200, self.res404 = _responses(self.req, [200, 404]) @@ -173,7 +171,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.assertIsNone(mw.process_spider_input(res402, self.spider)) -class TestHttpErrorMiddlewareIntegrational(TestCase): +class TestHttpErrorMiddlewareIntegrational(TrialTestCase): def setUp(self): self.mockserver = MockServer() self.mockserver.__enter__() diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index ed00c0a6b..ea45b7698 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,21 +1,17 @@ import warnings +from unittest import TestCase from urllib.parse import urlparse -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest - from scrapy.http import Request, Response from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, PortWarning, URLWarning from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -class TestOffsiteMiddleware(unittest.TestCase): - @inlineCallbacks +class TestOffsiteMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) - yield crawler.crawl(**self._get_spiderargs()) - self.spider = crawler.spider + self.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index a3c284484..9111e4c82 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,23 +1,21 @@ +from unittest import TestCase + from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware +from scrapy.spiders import Spider from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider -class TestUrlLengthMiddleware(unittest.TestCase): - @inlineCallbacks +class TestUrlLengthMiddleware(TestCase): def setUp(self): self.maxlength = 25 settings = Settings({"URLLENGTH_LIMIT": self.maxlength}) - crawler = get_crawler(NoRequestsSpider) - yield crawler.crawl() - self.spider = crawler.spider + crawler = get_crawler(Spider) + self.spider = crawler._create_spider("foo") self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 2117e65b1..eae744df5 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,10 +1,9 @@ import logging import sys +import unittest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure -from twisted.trial import unittest from scrapy.extensions import telnet from scrapy.utils.log import ( @@ -14,7 +13,6 @@ from scrapy.utils.log import ( failure_to_exc_info, ) from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class FailureToExcInfoTest(unittest.TestCase): @@ -62,7 +60,6 @@ class TopLevelFormatterTest(unittest.TestCase): class LogCounterHandlerTest(unittest.TestCase): - @inlineCallbacks def setUp(self): settings = {"LOG_LEVEL": "WARNING"} if not telnet.TWISTED_CONCH_AVAILABLE: @@ -71,8 +68,7 @@ class LogCounterHandlerTest(unittest.TestCase): self.logger = logging.getLogger("test") self.logger.setLevel(logging.NOTSET) self.logger.propagate = False - self.crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield self.crawler.crawl() + self.crawler = get_crawler(settings_dict=settings) self.handler = LogCounterHandler(self.crawler) self.logger.addHandler(self.handler) diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 9ca9faa0c..e6d1abe3f 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,12 +1,11 @@ import json +import unittest import warnings from hashlib import sha1 from typing import Dict, Mapping, Optional, Tuple, Union from weakref import WeakKeyDictionary import pytest -from twisted.internet.defer import inlineCallbacks -from twisted.trial import unittest from w3lib.url import canonicalize_url from scrapy.http import Request @@ -23,7 +22,6 @@ from scrapy.utils.request import ( request_to_curl, ) from scrapy.utils.test import get_crawler -from tests.spiders import NoRequestsSpider class UtilsRequestTest(unittest.TestCase): @@ -451,18 +449,15 @@ class BackwardCompatibilityTestCase(unittest.TestCase): ) self.assertEqual(fp, old_fp) - @inlineCallbacks def test_component_backward_compatibility(self): for request_object in REQUEST_OBJECTS_TO_TEST: with warnings.catch_warnings(): warnings.simplefilter("ignore") - crawler = get_crawler(NoRequestsSpider, prevent_warnings=False) - yield crawler.crawl() + crawler = get_crawler(prevent_warnings=False) fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) - @inlineCallbacks def test_custom_component_backward_compatibility(self): """Tests that the backward-compatible request fingerprinting class featured in the documentation is indeed backward compatible and does not cause a @@ -485,8 +480,7 @@ class BackwardCompatibilityTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) fp = crawler.request_fingerprinter.fingerprint(request_object) old_fp = request_fingerprint_2_6(request_object) self.assertEqual(fp.hex(), old_fp) @@ -494,11 +488,9 @@ class BackwardCompatibilityTestCase(unittest.TestCase): class RequestFingerprinterTestCase(unittest.TestCase): - @inlineCallbacks def test_default_implementation(self): with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(NoRequestsSpider, prevent_warnings=False) - yield crawler.crawl() + crawler = get_crawler(prevent_warnings=False) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -506,14 +498,12 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) - @inlineCallbacks def test_deprecated_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.6", } with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -521,14 +511,12 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertTrue(logged_warnings) - @inlineCallbacks def test_recommended_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", } with warnings.catch_warnings(record=True) as logged_warnings: - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("https://example.com") self.assertEqual( crawler.request_fingerprinter.fingerprint(request), @@ -536,18 +524,15 @@ class RequestFingerprinterTestCase(unittest.TestCase): ) self.assertFalse(logged_warnings) - @inlineCallbacks def test_unknown_implementation(self): settings = { "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.5", } with self.assertRaises(ValueError): - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + get_crawler(settings_dict=settings) class CustomRequestFingerprinterTestCase(unittest.TestCase): - @inlineCallbacks def test_include_headers(self): class RequestFingerprinter: def fingerprint(self, request): @@ -556,8 +541,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com", headers={"X-ID": "1"}) fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -565,7 +549,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) - @inlineCallbacks def test_dont_canonicalize(self): class RequestFingerprinter: cache = WeakKeyDictionary() @@ -580,8 +563,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com?a=1&a=2") fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -589,7 +571,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): fp2 = crawler.request_fingerprinter.fingerprint(r2) self.assertNotEqual(fp1, fp2) - @inlineCallbacks def test_meta(self): class RequestFingerprinter: def fingerprint(self, request): @@ -600,8 +581,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): settings = { "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) r1 = Request("http://www.example.com") fp1 = crawler.request_fingerprinter.fingerprint(r1) @@ -616,7 +596,6 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): self.assertNotEqual(fp2, fp4) self.assertEqual(fp2, fp3) - @inlineCallbacks def test_from_crawler(self): class RequestFingerprinter: @classmethod @@ -633,14 +612,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) - @inlineCallbacks def test_from_settings(self): class RequestFingerprinter: @classmethod @@ -657,14 +634,12 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) self.assertEqual(fingerprint, settings["FINGERPRINT"]) - @inlineCallbacks def test_from_crawler_and_settings(self): class RequestFingerprinter: # This method is ignored due to the presence of from_crawler @@ -686,8 +661,7 @@ class CustomRequestFingerprinterTestCase(unittest.TestCase): "REQUEST_FINGERPRINTER_CLASS": RequestFingerprinter, "FINGERPRINT": b"fingerprint", } - crawler = get_crawler(NoRequestsSpider, settings_dict=settings) - yield crawler.crawl() + crawler = get_crawler(settings_dict=settings) request = Request("http://www.example.com") fingerprint = crawler.request_fingerprinter.fingerprint(request) @@ -754,3 +728,7 @@ class RequestToCurlTest(unittest.TestCase): " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'" ) self._test_request(request_object, expected_curl_command) + + +if __name__ == "__main__": + unittest.main()