scrapy/tests/test_spiderloader/__init__.py

import contextlib
import shutil
import sys
import tempfile
import warnings
from pathlib import Path
from tempfile import mkdtemp
from unittest import mock

from twisted.trial import unittest
from zope.interface.verify import verifyObject

# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
# alone
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.interfaces import ISpiderLoader
from scrapy.settings import Settings
from scrapy.spiderloader import SpiderLoader

module_dir = Path(__file__).resolve().parent


def _copytree(source: Path, target: Path):
    with contextlib.suppress(shutil.Error):
        shutil.copytree(source, target)


class SpiderLoaderTest(unittest.TestCase):
    def setUp(self):
        orig_spiders_dir = module_dir / "test_spiders"
        self.tmpdir = Path(tempfile.mkdtemp())
        self.spiders_dir = self.tmpdir / "test_spiders_xxx"
        _copytree(orig_spiders_dir, self.spiders_dir)
        sys.path.append(str(self.tmpdir))
        settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})
        self.spider_loader = SpiderLoader.from_settings(settings)

    def tearDown(self):
        del self.spider_loader
        del sys.modules["test_spiders_xxx"]
        sys.path.remove(str(self.tmpdir))

    def test_interface(self):
        verifyObject(ISpiderLoader, self.spider_loader)

    def test_list(self):
        self.assertEqual(
            set(self.spider_loader.list()), {"spider1", "spider2", "spider3", "spider4"}
        )

    def test_load(self):
        spider1 = self.spider_loader.load("spider1")
        self.assertEqual(spider1.__name__, "Spider1")

    def test_find_by_request(self):
        self.assertEqual(
            self.spider_loader.find_by_request(Request("http://scrapy1.org/test")),
            ["spider1"],
        )
        self.assertEqual(
            self.spider_loader.find_by_request(Request("http://scrapy2.org/test")),
            ["spider2"],
        )
        self.assertEqual(
            set(self.spider_loader.find_by_request(Request("http://scrapy3.org/test"))),
            {"spider1", "spider2"},
        )
        self.assertEqual(
            self.spider_loader.find_by_request(Request("http://scrapy999.org/test")), []
        )
        self.assertEqual(
            self.spider_loader.find_by_request(Request("http://spider3.com")), []
        )
        self.assertEqual(
            self.spider_loader.find_by_request(Request("http://spider3.com/onlythis")),
            ["spider3"],
        )

    def test_load_spider_module(self):
        module = "tests.test_spiderloader.test_spiders.spider1"
        settings = Settings({"SPIDER_MODULES": [module]})
        self.spider_loader = SpiderLoader.from_settings(settings)
        assert len(self.spider_loader._spiders) == 1

    def test_load_spider_module_multiple(self):
        prefix = "tests.test_spiderloader.test_spiders."
        module = ",".join(prefix + s for s in ("spider1", "spider2"))
        settings = Settings({"SPIDER_MODULES": module})
        self.spider_loader = SpiderLoader.from_settings(settings)
        assert len(self.spider_loader._spiders) == 2

    def test_load_base_spider(self):
        module = "tests.test_spiderloader.test_spiders.spider0"
        settings = Settings({"SPIDER_MODULES": [module]})
        self.spider_loader = SpiderLoader.from_settings(settings)
        assert len(self.spider_loader._spiders) == 0

    def test_load_spider_module_from_addons(self):
        module = "tests.test_spiderloader.spiders_from_addons.spider0"

        class SpiderModuleAddon:
            @classmethod
            def update_pre_crawler_settings(cls, settings):
                settings.set(
                    "SPIDER_MODULES",
                    [module],
                    "project",
                )

        runner = CrawlerRunner({"ADDONS": {SpiderModuleAddon: 1}})

        crawler = runner.create_crawler("spider_from_addon")
        self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
        self.assertEqual(crawler.spidercls.name, "spider_from_addon")
        self.assertTrue(len(crawler.settings["SPIDER_MODULES"]) == 1)

    def test_crawler_runner_loading(self):
        module = "tests.test_spiderloader.test_spiders.spider1"
        runner = CrawlerRunner(
            {
                "SPIDER_MODULES": [module],
            }
        )

        self.assertRaisesRegex(
            KeyError, "Spider not found", runner.create_crawler, "spider2"
        )

        crawler = runner.create_crawler("spider1")
        self.assertTrue(issubclass(crawler.spidercls, scrapy.Spider))
        self.assertEqual(crawler.spidercls.name, "spider1")

    def test_bad_spider_modules_exception(self):
        module = "tests.test_spiderloader.test_spiders.doesnotexist"
        settings = Settings({"SPIDER_MODULES": [module]})
        self.assertRaises(ImportError, SpiderLoader.from_settings, settings)

    def test_bad_spider_modules_warning(self):
        with warnings.catch_warnings(record=True) as w:
            module = "tests.test_spiderloader.test_spiders.doesnotexist"
            settings = Settings(
                {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
            )
            spider_loader = SpiderLoader.from_settings(settings)
            if str(w[0].message).startswith("_SixMetaPathImporter"):
                # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
                # at least until all six versions we can import (including botocore.vendored.six)
                # are updated to 1.16.0+
                w.pop(0)
            self.assertIn("Could not load spiders from module", str(w[0].message))

            spiders = spider_loader.list()
            self.assertEqual(spiders, [])

    def test_syntax_error_exception(self):
        module = "tests.test_spiderloader.test_spiders.spider1"
        with mock.patch.object(SpiderLoader, "_load_spiders") as m:
            m.side_effect = SyntaxError
            settings = Settings({"SPIDER_MODULES": [module]})
            self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)

    def test_syntax_error_warning(self):
        with (
            warnings.catch_warnings(record=True) as w,
            mock.patch.object(SpiderLoader, "_load_spiders") as m,
        ):
            m.side_effect = SyntaxError
            module = "tests.test_spiderloader.test_spiders.spider1"
            settings = Settings(
                {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
            )
            spider_loader = SpiderLoader.from_settings(settings)
            if str(w[0].message).startswith("_SixMetaPathImporter"):
                # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
                # at least until all six versions we can import (including botocore.vendored.six)
                # are updated to 1.16.0+
                w.pop(0)
            self.assertIn("Could not load spiders from module", str(w[0].message))

            spiders = spider_loader.list()
            self.assertEqual(spiders, [])


class DuplicateSpiderNameLoaderTest(unittest.TestCase):
    def setUp(self):
        orig_spiders_dir = module_dir / "test_spiders"
        self.tmpdir = Path(mkdtemp())
        self.spiders_dir = self.tmpdir / "test_spiders_xxx"
        _copytree(orig_spiders_dir, self.spiders_dir)
        sys.path.append(str(self.tmpdir))
        self.settings = Settings({"SPIDER_MODULES": ["test_spiders_xxx"]})

    def tearDown(self):
        del sys.modules["test_spiders_xxx"]
        sys.path.remove(str(self.tmpdir))

    def test_dupename_warning(self):
        # copy 1 spider module so as to have duplicate spider name
        shutil.copyfile(
            self.tmpdir / "test_spiders_xxx" / "spider3.py",
            self.tmpdir / "test_spiders_xxx" / "spider3dupe.py",
        )

        with warnings.catch_warnings(record=True) as w:
            spider_loader = SpiderLoader.from_settings(self.settings)

            self.assertEqual(len(w), 1)
            msg = str(w[0].message)
            self.assertIn("several spiders with the same name", msg)
            self.assertIn("'spider3'", msg)
            self.assertTrue(msg.count("'spider3'") == 2)

            self.assertNotIn("'spider1'", msg)
            self.assertNotIn("'spider2'", msg)
            self.assertNotIn("'spider4'", msg)

            spiders = set(spider_loader.list())
            self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})

    def test_multiple_dupename_warning(self):
        # copy 2 spider modules so as to have duplicate spider name
        # This should issue 2 warning, 1 for each duplicate spider name
        shutil.copyfile(
            self.tmpdir / "test_spiders_xxx" / "spider1.py",
            self.tmpdir / "test_spiders_xxx" / "spider1dupe.py",
        )
        shutil.copyfile(
            self.tmpdir / "test_spiders_xxx" / "spider2.py",
            self.tmpdir / "test_spiders_xxx" / "spider2dupe.py",
        )

        with warnings.catch_warnings(record=True) as w:
            spider_loader = SpiderLoader.from_settings(self.settings)

            self.assertEqual(len(w), 1)
            msg = str(w[0].message)
            self.assertIn("several spiders with the same name", msg)
            self.assertIn("'spider1'", msg)
            self.assertTrue(msg.count("'spider1'") == 2)

            self.assertIn("'spider2'", msg)
            self.assertTrue(msg.count("'spider2'") == 2)

            self.assertNotIn("'spider3'", msg)
            self.assertNotIn("'spider4'", msg)

            spiders = set(spider_loader.list())
            self.assertEqual(spiders, {"spider1", "spider2", "spider3", "spider4"})