diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py new file mode 100644 index 000000000..3d5e17977 --- /dev/null +++ b/tests/test_command_crawl.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from pathlib import Path + +from tests.test_commands import TestCommandBase + + +class TestCrawlCommand(TestCommandBase): + def crawl(self, code, args=()): + Path(self.proj_mod_path, "spiders", "myspider.py").write_text( + code, encoding="utf-8" + ) + return self.proc("crawl", "myspider", *args) + + def get_log(self, code, args=()): + _, _, stderr = self.crawl(code, args=args) + return stderr + + def test_no_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('It works!') + return + yield +""" + log = self.get_log(spider_code) + assert "[myspider] DEBUG: It works!" in log + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(spider_code, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(spider_code, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py new file mode 100644 index 000000000..18ec81fed --- /dev/null +++ b/tests/test_command_genspider.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import os +from pathlib import Path + +from tests.test_commands import TestCommandBase, TestProjectBase + + +class TestGenspiderCommand(TestCommandBase): + def test_arguments(self): + # only pass one argument. spider script shouldn't be created + assert self.call("genspider", "test_name") == 2 + assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() + # pass two arguments . spider script should be created + assert self.call("genspider", "test_name", "test.com") == 0 + assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() + + def test_template(self, tplname="crawl"): + args = [f"--template={tplname}"] if tplname else [] + spname = "test_spider" + spmodule = f"{self.project_name}.spiders.{spname}" + p, out, err = self.proc("genspider", spname, "test.com", *args) + assert ( + f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" + in out + ) + assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() + modify_time_before = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) + p, out, err = self.proc("genspider", spname, "test.com", *args) + assert f"Spider {spname!r} already exists in module" in out + modify_time_after = ( + Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime + ) + assert modify_time_after == modify_time_before + + def test_template_basic(self): + self.test_template("basic") + + def test_template_csvfeed(self): + self.test_template("csvfeed") + + def test_template_xmlfeed(self): + self.test_template("xmlfeed") + + def test_list(self): + assert self.call("genspider", "--list") == 0 + + def test_dump(self): + assert self.call("genspider", "--dump=basic") == 0 + assert self.call("genspider", "-d", "basic") == 0 + + def test_same_name_as_project(self): + assert self.call("genspider", self.project_name) == 2 + assert not Path( + self.proj_mod_path, "spiders", f"{self.project_name}.py" + ).exists() + + def test_same_filename_as_existing_spider(self, force=False): + file_name = "example" + file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") + assert self.call("genspider", file_name, "example.com") == 0 + assert file_path.exists() + + # change name of spider but not its file name + with file_path.open("r+", encoding="utf-8") as spider_file: + file_data = spider_file.read() + file_data = file_data.replace('name = "example"', 'name = "renamed"') + spider_file.seek(0) + spider_file.write(file_data) + spider_file.truncate() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_data + + if force: + p, out, err = self.proc("genspider", "--force", file_name, "example.com") + assert ( + f"Created spider {file_name!r} using template 'basic' in module" in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + p, out, err = self.proc("genspider", file_name, "example.com") + assert f"{file_path.resolve()} already exists" in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before + + def test_same_filename_as_existing_spider_force(self): + self.test_same_filename_as_existing_spider(force=True) + + def test_url(self, url="test.com", domain="test.com"): + assert self.call("genspider", "--force", "test_name", url) == 0 + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == domain + ) + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == f"https://{domain}" + ) + + def test_url_schema(self): + self.test_url("https://test.com", "test.com") + + def test_template_start_urls( + self, url="test.com", expected="https://test.com", template="basic" + ): + assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 + assert ( + self.find_in_file( + Path(self.proj_mod_path, "spiders", "test_name.py"), + r"start_urls\s*=\s*\[['\"](.+)['\"]\]", + ).group(1) + == expected + ) + + def test_genspider_basic_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "basic") + self.test_template_start_urls("http://test.com", "http://test.com", "basic") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "basic" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "basic" + ) + + def test_genspider_crawl_start_urls(self): + self.test_template_start_urls("https://test.com", "https://test.com", "crawl") + self.test_template_start_urls("http://test.com", "http://test.com", "crawl") + self.test_template_start_urls( + "http://test.com/other/path", "http://test.com/other/path", "crawl" + ) + self.test_template_start_urls( + "test.com/other/path", "https://test.com/other/path", "crawl" + ) + self.test_template_start_urls("test.com", "https://test.com", "crawl") + + def test_genspider_xmlfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" + ) + self.test_template_start_urls( + "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" + ) + + def test_genspider_csvfeed_start_urls(self): + self.test_template_start_urls( + "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + self.test_template_start_urls( + "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" + ) + self.test_template_start_urls( + "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" + ) + + +class TestGenspiderStandaloneCommand(TestProjectBase): + def test_generate_standalone_spider(self): + self.call("genspider", "example", "example.com") + assert Path(self.temp_path, "example.py").exists() + + def test_same_name_as_existing_file(self, force=False): + file_name = "example" + file_path = Path(self.temp_path, file_name + ".py") + p, out, err = self.proc("genspider", file_name, "example.com") + assert f"Created spider {file_name!r} using template 'basic' " in out + assert file_path.exists() + modify_time_before = file_path.stat().st_mtime + file_contents_before = file_path.read_text(encoding="utf-8") + + if force: + # use different template to ensure contents were changed + p, out, err = self.proc( + "genspider", "--force", "-t", "crawl", file_name, "example.com" + ) + assert f"Created spider {file_name!r} using template 'crawl' " in out + modify_time_after = file_path.stat().st_mtime + assert modify_time_after != modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after != file_contents_before + else: + p, out, err = self.proc("genspider", file_name, "example.com") + assert ( + f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" + in out + ) + modify_time_after = file_path.stat().st_mtime + assert modify_time_after == modify_time_before + file_contents_after = file_path.read_text(encoding="utf-8") + assert file_contents_after == file_contents_before + + def test_same_name_as_existing_file_force(self): + self.test_same_name_as_existing_file(force=True) diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py new file mode 100644 index 000000000..664de16f8 --- /dev/null +++ b/tests/test_command_runspider.py @@ -0,0 +1,375 @@ +from __future__ import annotations + +import inspect +import platform +import sys +from contextlib import contextmanager +from pathlib import Path +from tempfile import TemporaryDirectory, mkdtemp +from typing import TYPE_CHECKING +from unittest import skipIf + +import pytest +from twisted.trial import unittest + +from tests.test_commands import TestCommandBase +from tests.test_crawler import ExceptionSpider, NoRequestsSpider + +if TYPE_CHECKING: + from collections.abc import Iterator + + +class TestRunSpiderCommand(TestCommandBase): + spider_filename = "myspider.py" + + debug_log_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug("It Works!") + return + yield +""" + + badspider = """ +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + async def start(self): + raise Exception("oops!") + yield + """ + + @contextmanager + def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: + with TemporaryDirectory() as tmpdir: + if name: + fname = Path(tmpdir, name).resolve() + else: + fname = Path(tmpdir, self.spider_filename).resolve() + fname.write_text(content, encoding="utf-8") + yield str(fname) + + def runspider(self, code, name=None, args=()): + with self._create_file(code, name) as fname: + return self.proc("runspider", fname, *args) + + def get_log(self, code, name=None, args=()): + p, stdout, stderr = self.runspider(code, name, args=args) + return stderr + + def test_runspider(self): + log = self.get_log(self.debug_log_spider) + assert "DEBUG: It Works!" in log + assert "INFO: Spider opened" in log + assert "INFO: Closing spider (finished)" in log + assert "INFO: Spider closed (finished)" in log + + def test_run_fail_spider(self): + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(ExceptionSpider) + ) + ret = proc.returncode + assert ret != 0 + + def test_run_good_spider(self): + proc, _, _ = self.runspider( + "import scrapy\n" + inspect.getsource(NoRequestsSpider) + ) + ret = proc.returncode + assert ret == 0 + + def test_runspider_log_level(self): + log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) + assert "DEBUG: It Works!" not in log + assert "INFO: Spider opened" in log + + def test_runspider_dnscache_disabled(self): + # see https://github.com/scrapy/scrapy/issues/2811 + # The spider below should not be able to connect to localhost:12345, + # which is intended, + # but this should not be because of DNS lookup error + # assumption: localhost will resolve in all cases (true?) + dnscache_spider = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + start_urls = ['http://localhost:12345'] + + def parse(self, response): + return {'test': 'value'} +""" + log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) + assert "DNSLookupError" not in log + assert "INFO: Spider opened" in log + + def test_runspider_log_short_names(self): + log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) + assert "[myspider] DEBUG: It Works!" in log1 + assert "[scrapy]" in log1 + assert "[scrapy.core.engine]" not in log1 + + log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) + assert "[myspider] DEBUG: It Works!" in log2 + assert "[scrapy]" not in log2 + assert "[scrapy.core.engine]" in log2 + + def test_runspider_no_spider_found(self): + log = self.get_log("from scrapy.spiders import Spider\n") + assert "No spider found in file" in log + + def test_runspider_file_not_found(self): + _, _, log = self.proc("runspider", "some_non_existent_file") + assert "File not found: some_non_existent_file" in log + + def test_runspider_unable_to_load(self): + log = self.get_log("", name="myspider.txt") + assert "Unable to load" in log + + def test_start_errors(self): + log = self.get_log(self.badspider, name="badspider.py") + assert "start" in log + assert "badspider.py" in log, log + + def test_asyncio_enabled_true(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_default(self): + log = self.get_log(self.debug_log_spider, args=[]) + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_asyncio_enabled_false(self): + log = self.get_log( + self.debug_log_spider, + args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], + ) + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + @pytest.mark.requires_uvloop + def test_custom_asyncio_loop_enabled_true(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "-s", + "ASYNCIO_EVENT_LOOP=uvloop.Loop", + ], + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + def test_custom_asyncio_loop_enabled_false(self): + log = self.get_log( + self.debug_log_spider, + args=[ + "-s", + "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", + ], + ) + import asyncio + + if sys.platform != "win32": + loop = asyncio.new_event_loop() + else: + loop = asyncio.SelectorEventLoop() + assert ( + f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" + in log + ) + + def test_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "example.json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log + + def test_overwrite_output(self): + spider_code = """ +import json +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug( + 'FEEDS: {}'.format( + json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) + ) + ) + return + yield +""" + Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") + args = ["-O", "example.json"] + log = self.get_log(spider_code, args=args) + assert ( + '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' + in log + ) + with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: + first_line = f2.readline() + assert first_line != "not empty" + + def test_output_and_overwrite_output(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + return + yield +""" + args = ["-o", "example1.json", "-O", "example2.json"] + log = self.get_log(spider_code, args=args) + assert ( + "error: Please use only one of -o/--output and -O/--overwrite-output" in log + ) + + def test_output_stdout(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + async def start(self): + self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) + return + yield +""" + args = ["-o", "-:json"] + log = self.get_log(spider_code, args=args) + assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log + + @skipIf(platform.system() == "Windows", reason="Linux only") + def test_absolute_path_linux(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}/output1.json:json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" + in log + ) + + args = ["-o", f"{temp_dir}/output2.json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" + in log + ) + + @skipIf(platform.system() != "Windows", reason="Windows only") + def test_absolute_path_windows(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + start_urls = ["data:,"] + + def parse(self, response): + yield {"hello": "world"} + """ + temp_dir = mkdtemp() + + args = ["-o", f"{temp_dir}\\output1.json:json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" + in log + ) + + args = ["-o", f"{temp_dir}\\output2.json"] + log = self.get_log(spider_code, args=args) + assert ( + f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" + in log + ) + + def test_args_change_settings(self): + spider_code = """ +import scrapy + +class MySpider(scrapy.Spider): + name = 'myspider' + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield +""" + args = ["-a", "foo=42"] + log = self.get_log(spider_code, args=args) + assert "Spider closed (finished)" in log + assert "The value of FOO is 42" in log + + +class TestWindowsRunSpiderCommand(TestRunSpiderCommand): + spider_filename = "myspider.pyw" + + def setUp(self): + if platform.system() != "Windows": + raise unittest.SkipTest("Windows required for .pyw files") + return super().setUp() + + def test_start_errors(self): + log = self.get_log(self.badspider, name="badspider.pyw") + assert "start" in log + assert "badspider.pyw" in log + + def test_runspider_unable_to_load(self): + raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py new file mode 100644 index 000000000..08bf9b0fd --- /dev/null +++ b/tests/test_command_startproject.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import os +import subprocess +import sys +from contextlib import contextmanager +from itertools import chain +from pathlib import Path +from shutil import copytree +from stat import S_IWRITE as ANYONE_WRITE_PERMISSION +from tempfile import mkdtemp + +import scrapy +from scrapy.commands.startproject import IGNORE +from tests.test_commands import TestProjectBase + + +class TestStartprojectCommand(TestProjectBase): + def test_startproject(self): + p, out, err = self.proc("startproject", self.project_name) + print(out) + print(err, file=sys.stderr) + assert p.returncode == 0 + + assert Path(self.proj_path, "scrapy.cfg").exists() + assert Path(self.proj_path, "testproject").exists() + assert Path(self.proj_mod_path, "__init__.py").exists() + assert Path(self.proj_mod_path, "items.py").exists() + assert Path(self.proj_mod_path, "pipelines.py").exists() + assert Path(self.proj_mod_path, "settings.py").exists() + assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() + + assert self.call("startproject", self.project_name) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + + def test_startproject_with_project_dir(self): + project_dir = mkdtemp() + assert self.call("startproject", self.project_name, project_dir) == 0 + + assert Path(project_dir, "scrapy.cfg").exists() + assert Path(project_dir, "testproject").exists() + assert Path(project_dir, self.project_name, "__init__.py").exists() + assert Path(project_dir, self.project_name, "items.py").exists() + assert Path(project_dir, self.project_name, "pipelines.py").exists() + assert Path(project_dir, self.project_name, "settings.py").exists() + assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() + + assert self.call("startproject", self.project_name, project_dir + "2") == 0 + + assert self.call("startproject", self.project_name, project_dir) == 1 + assert self.call("startproject", self.project_name + "2", project_dir) == 1 + assert self.call("startproject", "wrong---project---name") == 1 + assert self.call("startproject", "sys") == 1 + assert self.call("startproject") == 2 + assert ( + self.call("startproject", self.project_name, project_dir, "another_params") + == 2 + ) + + def test_existing_project_dir(self): + project_dir = mkdtemp() + project_name = self.project_name + "_existing" + project_path = Path(project_dir, project_name) + project_path.mkdir() + + p, out, err = self.proc("startproject", project_name, cwd=project_dir) + print(out) + print(err, file=sys.stderr) + assert p.returncode == 0 + + assert Path(project_path, "scrapy.cfg").exists() + assert Path(project_path, project_name).exists() + assert Path(project_path, project_name, "__init__.py").exists() + assert Path(project_path, project_name, "items.py").exists() + assert Path(project_path, project_name, "pipelines.py").exists() + assert Path(project_path, project_name, "settings.py").exists() + assert Path(project_path, project_name, "spiders", "__init__.py").exists() + + +def get_permissions_dict( + path: str | os.PathLike, renamings=None, ignore=None +) -> dict[str, str]: + def get_permissions(path: Path) -> str: + return oct(path.stat().st_mode) + + path_obj = Path(path) + + renamings = renamings or () + permissions_dict = { + ".": get_permissions(path_obj), + } + for root, dirs, files in os.walk(path_obj): + nodes = list(chain(dirs, files)) + if ignore: + ignored_names = ignore(root, nodes) + nodes = [node for node in nodes if node not in ignored_names] + for node in nodes: + absolute_path = Path(root, node) + relative_path = str(absolute_path.relative_to(path)) + for search_string, replacement in renamings: + relative_path = relative_path.replace(search_string, replacement) + permissions = get_permissions(absolute_path) + permissions_dict[relative_path] = permissions + return permissions_dict + + +class TestStartprojectTemplates(TestProjectBase): + maxDiff = None + + def setUp(self): + super().setUp() + self.tmpl = str(Path(self.temp_path, "templates")) + self.tmpl_proj = str(Path(self.tmpl, "project")) + + def test_startproject_template_override(self): + copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) + Path(self.tmpl_proj, "root_template").write_bytes(b"") + assert Path(self.tmpl_proj, "root_template").exists() + + args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] + p, out, err = self.proc("startproject", self.project_name, *args) + assert ( + f"New Scrapy project '{self.project_name}', using template directory" in out + ) + assert self.tmpl_proj in out + assert Path(self.proj_path, "root_template").exists() + + def test_startproject_permissions_from_writable(self): + """Check that generated files have the right permissions when the + template folder has the same permissions as in the project, i.e. + everything is writable.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject1" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_from_read_only(self): + """Check that generated files have the right permissions when the + template folder has been made read-only, which is something that some + systems do. + + See https://github.com/scrapy/scrapy/pull/4604 + """ + scrapy_path = scrapy.__path__[0] + templates_dir = Path(scrapy_path, "templates") + project_template = Path(templates_dir, "project") + project_name = "startproject2" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + def _make_read_only(path: Path): + current_permissions = path.stat().st_mode + path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) + + read_only_templates_dir = str(Path(mkdtemp()) / "templates") + copytree(templates_dir, read_only_templates_dir) + + for root, dirs, files in os.walk(read_only_templates_dir): + for node in chain(dirs, files): + _make_read_only(Path(root, node)) + + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + "--set", + f"TEMPLATES_DIR={read_only_templates_dir}", + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_unchanged_in_destination(self): + """Check that preexisting folders and files in the destination folder + do not see their permissions modified.""" + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "startproject3" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + destination = mkdtemp() + project_dir = Path(destination, project_name) + + existing_nodes = { + oct(permissions)[2:] + extension: permissions + for extension in ("", ".d") + for permissions in ( + 0o444, + 0o555, + 0o644, + 0o666, + 0o755, + 0o777, + ) + } + project_dir.mkdir() + for node, permissions in existing_nodes.items(): + path = project_dir / node + if node.endswith(".d"): + path.mkdir(mode=permissions) + else: + path.touch(mode=permissions) + expected_permissions[node] = oct(path.stat().st_mode) + + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ".", + ), + cwd=project_dir, + env=self.env, + ) + process.wait() + + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions + + def test_startproject_permissions_umask_022(self): + """Check that generated files have the right permissions when the + system uses a umask value that causes new files to have different + permissions than those from the template folder.""" + + @contextmanager + def umask(new_mask): + cur_mask = os.umask(new_mask) + yield + os.umask(cur_mask) + + scrapy_path = scrapy.__path__[0] + project_template = Path(scrapy_path, "templates", "project") + project_name = "umaskproject" + renamings = ( + ("module", project_name), + (".tmpl", ""), + ) + expected_permissions = get_permissions_dict( + project_template, + renamings, + IGNORE, + ) + + with umask(0o002): + destination = mkdtemp() + process = subprocess.Popen( + ( + sys.executable, + "-m", + "scrapy.cmdline", + "startproject", + project_name, + ), + cwd=destination, + env=self.env, + ) + process.wait() + + project_dir = Path(destination, project_name) + actual_permissions = get_permissions_dict(project_dir) + + assert actual_permissions == expected_permissions diff --git a/tests/test_commands.py b/tests/test_commands.py index 16af97842..6e59f561d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1,38 +1,29 @@ from __future__ import annotations import argparse -import inspect import json -import os -import platform import re import subprocess import sys -from contextlib import contextmanager from io import StringIO -from itertools import chain from pathlib import Path -from shutil import copytree, rmtree -from stat import S_IWRITE as ANYONE_WRITE_PERMISSION -from tempfile import TemporaryDirectory, TemporaryFile, mkdtemp +from shutil import rmtree +from tempfile import TemporaryFile, mkdtemp from threading import Timer from typing import TYPE_CHECKING -from unittest import mock, skipIf +from unittest import mock -import pytest from twisted.trial import unittest import scrapy from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view -from scrapy.commands.startproject import IGNORE from scrapy.settings import Settings from scrapy.utils.python import to_unicode from scrapy.utils.test import get_testenv -from tests.test_crawler import ExceptionSpider, NoRequestsSpider if TYPE_CHECKING: - from collections.abc import Iterator + import os class TestCommandSettings: @@ -125,309 +116,6 @@ class TestProjectBase(unittest.TestCase): return None -class TestStartprojectCommand(TestProjectBase): - def test_startproject(self): - p, out, err = self.proc("startproject", self.project_name) - print(out) - print(err, file=sys.stderr) - assert p.returncode == 0 - - assert Path(self.proj_path, "scrapy.cfg").exists() - assert Path(self.proj_path, "testproject").exists() - assert Path(self.proj_mod_path, "__init__.py").exists() - assert Path(self.proj_mod_path, "items.py").exists() - assert Path(self.proj_mod_path, "pipelines.py").exists() - assert Path(self.proj_mod_path, "settings.py").exists() - assert Path(self.proj_mod_path, "spiders", "__init__.py").exists() - - assert self.call("startproject", self.project_name) == 1 - assert self.call("startproject", "wrong---project---name") == 1 - assert self.call("startproject", "sys") == 1 - - def test_startproject_with_project_dir(self): - project_dir = mkdtemp() - assert self.call("startproject", self.project_name, project_dir) == 0 - - assert Path(project_dir, "scrapy.cfg").exists() - assert Path(project_dir, "testproject").exists() - assert Path(project_dir, self.project_name, "__init__.py").exists() - assert Path(project_dir, self.project_name, "items.py").exists() - assert Path(project_dir, self.project_name, "pipelines.py").exists() - assert Path(project_dir, self.project_name, "settings.py").exists() - assert Path(project_dir, self.project_name, "spiders", "__init__.py").exists() - - assert self.call("startproject", self.project_name, project_dir + "2") == 0 - - assert self.call("startproject", self.project_name, project_dir) == 1 - assert self.call("startproject", self.project_name + "2", project_dir) == 1 - assert self.call("startproject", "wrong---project---name") == 1 - assert self.call("startproject", "sys") == 1 - assert self.call("startproject") == 2 - assert ( - self.call("startproject", self.project_name, project_dir, "another_params") - == 2 - ) - - def test_existing_project_dir(self): - project_dir = mkdtemp() - project_name = self.project_name + "_existing" - project_path = Path(project_dir, project_name) - project_path.mkdir() - - p, out, err = self.proc("startproject", project_name, cwd=project_dir) - print(out) - print(err, file=sys.stderr) - assert p.returncode == 0 - - assert Path(project_path, "scrapy.cfg").exists() - assert Path(project_path, project_name).exists() - assert Path(project_path, project_name, "__init__.py").exists() - assert Path(project_path, project_name, "items.py").exists() - assert Path(project_path, project_name, "pipelines.py").exists() - assert Path(project_path, project_name, "settings.py").exists() - assert Path(project_path, project_name, "spiders", "__init__.py").exists() - - -def get_permissions_dict( - path: str | os.PathLike, renamings=None, ignore=None -) -> dict[str, str]: - def get_permissions(path: Path) -> str: - return oct(path.stat().st_mode) - - path_obj = Path(path) - - renamings = renamings or () - permissions_dict = { - ".": get_permissions(path_obj), - } - for root, dirs, files in os.walk(path_obj): - nodes = list(chain(dirs, files)) - if ignore: - ignored_names = ignore(root, nodes) - nodes = [node for node in nodes if node not in ignored_names] - for node in nodes: - absolute_path = Path(root, node) - relative_path = str(absolute_path.relative_to(path)) - for search_string, replacement in renamings: - relative_path = relative_path.replace(search_string, replacement) - permissions = get_permissions(absolute_path) - permissions_dict[relative_path] = permissions - return permissions_dict - - -class TestStartprojectTemplates(TestProjectBase): - maxDiff = None - - def setUp(self): - super().setUp() - self.tmpl = str(Path(self.temp_path, "templates")) - self.tmpl_proj = str(Path(self.tmpl, "project")) - - def test_startproject_template_override(self): - copytree(Path(scrapy.__path__[0], "templates"), self.tmpl) - Path(self.tmpl_proj, "root_template").write_bytes(b"") - assert Path(self.tmpl_proj, "root_template").exists() - - args = ["--set", f"TEMPLATES_DIR={self.tmpl}"] - p, out, err = self.proc("startproject", self.project_name, *args) - assert ( - f"New Scrapy project '{self.project_name}', using template directory" in out - ) - assert self.tmpl_proj in out - assert Path(self.proj_path, "root_template").exists() - - def test_startproject_permissions_from_writable(self): - """Check that generated files have the right permissions when the - template folder has the same permissions as in the project, i.e. - everything is writable.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject1" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_from_read_only(self): - """Check that generated files have the right permissions when the - template folder has been made read-only, which is something that some - systems do. - - See https://github.com/scrapy/scrapy/pull/4604 - """ - scrapy_path = scrapy.__path__[0] - templates_dir = Path(scrapy_path, "templates") - project_template = Path(templates_dir, "project") - project_name = "startproject2" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - def _make_read_only(path: Path): - current_permissions = path.stat().st_mode - path.chmod(current_permissions & ~ANYONE_WRITE_PERMISSION) - - read_only_templates_dir = str(Path(mkdtemp()) / "templates") - copytree(templates_dir, read_only_templates_dir) - - for root, dirs, files in os.walk(read_only_templates_dir): - for node in chain(dirs, files): - _make_read_only(Path(root, node)) - - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - "--set", - f"TEMPLATES_DIR={read_only_templates_dir}", - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_unchanged_in_destination(self): - """Check that preexisting folders and files in the destination folder - do not see their permissions modified.""" - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "startproject3" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - destination = mkdtemp() - project_dir = Path(destination, project_name) - - existing_nodes = { - oct(permissions)[2:] + extension: permissions - for extension in ("", ".d") - for permissions in ( - 0o444, - 0o555, - 0o644, - 0o666, - 0o755, - 0o777, - ) - } - project_dir.mkdir() - for node, permissions in existing_nodes.items(): - path = project_dir / node - if node.endswith(".d"): - path.mkdir(mode=permissions) - else: - path.touch(mode=permissions) - expected_permissions[node] = oct(path.stat().st_mode) - - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ".", - ), - cwd=project_dir, - env=self.env, - ) - process.wait() - - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - def test_startproject_permissions_umask_022(self): - """Check that generated files have the right permissions when the - system uses a umask value that causes new files to have different - permissions than those from the template folder.""" - - @contextmanager - def umask(new_mask): - cur_mask = os.umask(new_mask) - yield - os.umask(cur_mask) - - scrapy_path = scrapy.__path__[0] - project_template = Path(scrapy_path, "templates", "project") - project_name = "umaskproject" - renamings = ( - ("module", project_name), - (".tmpl", ""), - ) - expected_permissions = get_permissions_dict( - project_template, - renamings, - IGNORE, - ) - - with umask(0o002): - destination = mkdtemp() - process = subprocess.Popen( - ( - sys.executable, - "-m", - "scrapy.cmdline", - "startproject", - project_name, - ), - cwd=destination, - env=self.env, - ) - process.wait() - - project_dir = Path(destination, project_name) - actual_permissions = get_permissions_dict(project_dir) - - assert actual_permissions == expected_permissions - - class TestCommandBase(TestProjectBase): def setUp(self): super().setUp() @@ -436,208 +124,6 @@ class TestCommandBase(TestProjectBase): self.env["SCRAPY_SETTINGS_MODULE"] = f"{self.project_name}.settings" -class TestGenspiderCommand(TestCommandBase): - def test_arguments(self): - # only pass one argument. spider script shouldn't be created - assert self.call("genspider", "test_name") == 2 - assert not Path(self.proj_mod_path, "spiders", "test_name.py").exists() - # pass two arguments . spider script should be created - assert self.call("genspider", "test_name", "test.com") == 0 - assert Path(self.proj_mod_path, "spiders", "test_name.py").exists() - - def test_template(self, tplname="crawl"): - args = [f"--template={tplname}"] if tplname else [] - spname = "test_spider" - spmodule = f"{self.project_name}.spiders.{spname}" - p, out, err = self.proc("genspider", spname, "test.com", *args) - assert ( - f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}" - in out - ) - assert Path(self.proj_mod_path, "spiders", "test_spider.py").exists() - modify_time_before = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - p, out, err = self.proc("genspider", spname, "test.com", *args) - assert f"Spider {spname!r} already exists in module" in out - modify_time_after = ( - Path(self.proj_mod_path, "spiders", "test_spider.py").stat().st_mtime - ) - assert modify_time_after == modify_time_before - - def test_template_basic(self): - self.test_template("basic") - - def test_template_csvfeed(self): - self.test_template("csvfeed") - - def test_template_xmlfeed(self): - self.test_template("xmlfeed") - - def test_list(self): - assert self.call("genspider", "--list") == 0 - - def test_dump(self): - assert self.call("genspider", "--dump=basic") == 0 - assert self.call("genspider", "-d", "basic") == 0 - - def test_same_name_as_project(self): - assert self.call("genspider", self.project_name) == 2 - assert not Path( - self.proj_mod_path, "spiders", f"{self.project_name}.py" - ).exists() - - def test_same_filename_as_existing_spider(self, force=False): - file_name = "example" - file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py") - assert self.call("genspider", file_name, "example.com") == 0 - assert file_path.exists() - - # change name of spider but not its file name - with file_path.open("r+", encoding="utf-8") as spider_file: - file_data = spider_file.read() - file_data = file_data.replace('name = "example"', 'name = "renamed"') - spider_file.seek(0) - spider_file.write(file_data) - spider_file.truncate() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_data - - if force: - p, out, err = self.proc("genspider", "--force", file_name, "example.com") - assert ( - f"Created spider {file_name!r} using template 'basic' in module" in out - ) - modify_time_after = file_path.stat().st_mtime - assert modify_time_after != modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after != file_contents_before - else: - p, out, err = self.proc("genspider", file_name, "example.com") - assert f"{file_path.resolve()} already exists" in out - modify_time_after = file_path.stat().st_mtime - assert modify_time_after == modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after == file_contents_before - - def test_same_filename_as_existing_spider_force(self): - self.test_same_filename_as_existing_spider(force=True) - - def test_url(self, url="test.com", domain="test.com"): - assert self.call("genspider", "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == domain - ) - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == f"https://{domain}" - ) - - def test_url_schema(self): - self.test_url("https://test.com", "test.com") - - def test_template_start_urls( - self, url="test.com", expected="https://test.com", template="basic" - ): - assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0 - assert ( - self.find_in_file( - Path(self.proj_mod_path, "spiders", "test_name.py"), - r"start_urls\s*=\s*\[['\"](.+)['\"]\]", - ).group(1) - == expected - ) - - def test_genspider_basic_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "basic") - self.test_template_start_urls("http://test.com", "http://test.com", "basic") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "basic" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "basic" - ) - - def test_genspider_crawl_start_urls(self): - self.test_template_start_urls("https://test.com", "https://test.com", "crawl") - self.test_template_start_urls("http://test.com", "http://test.com", "crawl") - self.test_template_start_urls( - "http://test.com/other/path", "http://test.com/other/path", "crawl" - ) - self.test_template_start_urls( - "test.com/other/path", "https://test.com/other/path", "crawl" - ) - self.test_template_start_urls("test.com", "https://test.com", "crawl") - - def test_genspider_xmlfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed" - ) - self.test_template_start_urls( - "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed" - ) - - def test_genspider_csvfeed_start_urls(self): - self.test_template_start_urls( - "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - self.test_template_start_urls( - "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed" - ) - self.test_template_start_urls( - "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed" - ) - - -class TestGenspiderStandaloneCommand(TestProjectBase): - def test_generate_standalone_spider(self): - self.call("genspider", "example", "example.com") - assert Path(self.temp_path, "example.py").exists() - - def test_same_name_as_existing_file(self, force=False): - file_name = "example" - file_path = Path(self.temp_path, file_name + ".py") - p, out, err = self.proc("genspider", file_name, "example.com") - assert f"Created spider {file_name!r} using template 'basic' " in out - assert file_path.exists() - modify_time_before = file_path.stat().st_mtime - file_contents_before = file_path.read_text(encoding="utf-8") - - if force: - # use different template to ensure contents were changed - p, out, err = self.proc( - "genspider", "--force", "-t", "crawl", file_name, "example.com" - ) - assert f"Created spider {file_name!r} using template 'crawl' " in out - modify_time_after = file_path.stat().st_mtime - assert modify_time_after != modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after != file_contents_before - else: - p, out, err = self.proc("genspider", file_name, "example.com") - assert ( - f"{Path(self.temp_path, file_name + '.py').resolve()} already exists" - in out - ) - modify_time_after = file_path.stat().st_mtime - assert modify_time_after == modify_time_before - file_contents_after = file_path.read_text(encoding="utf-8") - assert file_contents_after == file_contents_before - - def test_same_name_as_existing_file_force(self): - self.test_same_name_as_existing_file(force=True) - - class TestMiscCommands(TestCommandBase): def test_list(self): assert self.call("list") == 0 @@ -661,362 +147,6 @@ Unknown command: abc assert out.getvalue().strip() == message.strip() -class TestRunSpiderCommand(TestCommandBase): - spider_filename = "myspider.py" - - debug_log_spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug("It Works!") - return - yield -""" - - badspider = """ -import scrapy - -class BadSpider(scrapy.Spider): - name = "bad" - async def start(self): - raise Exception("oops!") - yield - """ - - @contextmanager - def _create_file(self, content: str, name: str | None = None) -> Iterator[str]: - with TemporaryDirectory() as tmpdir: - if name: - fname = Path(tmpdir, name).resolve() - else: - fname = Path(tmpdir, self.spider_filename).resolve() - fname.write_text(content, encoding="utf-8") - yield str(fname) - - def runspider(self, code, name=None, args=()): - with self._create_file(code, name) as fname: - return self.proc("runspider", fname, *args) - - def get_log(self, code, name=None, args=()): - p, stdout, stderr = self.runspider(code, name, args=args) - return stderr - - def test_runspider(self): - log = self.get_log(self.debug_log_spider) - assert "DEBUG: It Works!" in log - assert "INFO: Spider opened" in log - assert "INFO: Closing spider (finished)" in log - assert "INFO: Spider closed (finished)" in log - - def test_run_fail_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(ExceptionSpider) - ) - ret = proc.returncode - assert ret != 0 - - def test_run_good_spider(self): - proc, _, _ = self.runspider( - "import scrapy\n" + inspect.getsource(NoRequestsSpider) - ) - ret = proc.returncode - assert ret == 0 - - def test_runspider_log_level(self): - log = self.get_log(self.debug_log_spider, args=("-s", "LOG_LEVEL=INFO")) - assert "DEBUG: It Works!" not in log - assert "INFO: Spider opened" in log - - def test_runspider_dnscache_disabled(self): - # see https://github.com/scrapy/scrapy/issues/2811 - # The spider below should not be able to connect to localhost:12345, - # which is intended, - # but this should not be because of DNS lookup error - # assumption: localhost will resolve in all cases (true?) - dnscache_spider = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - start_urls = ['http://localhost:12345'] - - def parse(self, response): - return {'test': 'value'} -""" - log = self.get_log(dnscache_spider, args=("-s", "DNSCACHE_ENABLED=False")) - assert "DNSLookupError" not in log - assert "INFO: Spider opened" in log - - def test_runspider_log_short_names(self): - log1 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=1")) - assert "[myspider] DEBUG: It Works!" in log1 - assert "[scrapy]" in log1 - assert "[scrapy.core.engine]" not in log1 - - log2 = self.get_log(self.debug_log_spider, args=("-s", "LOG_SHORT_NAMES=0")) - assert "[myspider] DEBUG: It Works!" in log2 - assert "[scrapy]" not in log2 - assert "[scrapy.core.engine]" in log2 - - def test_runspider_no_spider_found(self): - log = self.get_log("from scrapy.spiders import Spider\n") - assert "No spider found in file" in log - - def test_runspider_file_not_found(self): - _, _, log = self.proc("runspider", "some_non_existent_file") - assert "File not found: some_non_existent_file" in log - - def test_runspider_unable_to_load(self): - log = self.get_log("", name="myspider.txt") - assert "Unable to load" in log - - def test_start_errors(self): - log = self.get_log(self.badspider, name="badspider.py") - assert "start" in log - assert "badspider.py" in log, log - - def test_asyncio_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_asyncio_enabled_default(self): - log = self.get_log(self.debug_log_spider, args=[]) - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - - def test_asyncio_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=["-s", "TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor"], - ) - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - - @pytest.mark.requires_uvloop - def test_custom_asyncio_loop_enabled_true(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - "-s", - "ASYNCIO_EVENT_LOOP=uvloop.Loop", - ], - ) - assert "Using asyncio event loop: uvloop.Loop" in log - - def test_custom_asyncio_loop_enabled_false(self): - log = self.get_log( - self.debug_log_spider, - args=[ - "-s", - "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor", - ], - ) - import asyncio - - if sys.platform != "win32": - loop = asyncio.new_event_loop() - else: - loop = asyncio.SelectorEventLoop() - assert ( - f"Using asyncio event loop: {loop.__module__}.{loop.__class__.__name__}" - in log - ) - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) - ) - ) - return - yield -""" - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' - in log - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - assert first_line != "not empty" - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - return - yield -""" - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - assert ( - "error: Please use only one of -o/--output and -O/--overwrite-output" in log - ) - - def test_output_stdout(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "-:json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}" in log - - @skipIf(platform.system() == "Windows", reason="Linux only") - def test_absolute_path_linux(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}/output1.json:json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output1.json" - in log - ) - - args = ["-o", f"{temp_dir}/output2.json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}/output2.json" - in log - ) - - @skipIf(platform.system() != "Windows", reason="Windows only") - def test_absolute_path_windows(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - start_urls = ["data:,"] - - def parse(self, response): - yield {"hello": "world"} - """ - temp_dir = mkdtemp() - - args = ["-o", f"{temp_dir}\\output1.json:json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output1.json" - in log - ) - - args = ["-o", f"{temp_dir}\\output2.json"] - log = self.get_log(spider_code, args=args) - assert ( - f"[scrapy.extensions.feedexport] INFO: Stored json feed (1 items) in: {temp_dir}\\output2.json" - in log - ) - - def test_args_change_settings(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = super().from_crawler(crawler, *args, **kwargs) - spider.settings.set("FOO", kwargs.get("foo")) - return spider - - async def start(self): - self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") - return - yield -""" - args = ["-a", "foo=42"] - log = self.get_log(spider_code, args=args) - assert "Spider closed (finished)" in log - assert "The value of FOO is 42" in log - - -class TestWindowsRunSpiderCommand(TestRunSpiderCommand): - spider_filename = "myspider.pyw" - - def setUp(self): - if platform.system() != "Windows": - raise unittest.SkipTest("Windows required for .pyw files") - return super().setUp() - - def test_start_errors(self): - log = self.get_log(self.badspider, name="badspider.pyw") - assert "start" in log - assert "badspider.pyw" in log - - def test_runspider_unable_to_load(self): - raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") - - class TestBenchCommand(TestCommandBase): def test_run(self): _, _, log = self.proc( @@ -1042,94 +172,6 @@ class TestViewCommand(TestCommandBase): assert "URL using the Scrapy downloader and show its" in command.long_desc() -class TestCrawlCommand(TestCommandBase): - def crawl(self, code, args=()): - Path(self.proj_mod_path, "spiders", "myspider.py").write_text( - code, encoding="utf-8" - ) - return self.proc("crawl", "myspider", *args) - - def get_log(self, code, args=()): - _, _, stderr = self.crawl(code, args=args) - return stderr - - def test_no_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('It works!') - return - yield -""" - log = self.get_log(spider_code) - assert "[myspider] DEBUG: It works!" in log - - def test_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS'))) - return - yield -""" - args = ["-o", "example.json"] - log = self.get_log(spider_code, args=args) - assert "[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}" in log - - def test_overwrite_output(self): - spider_code = """ -import json -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - self.logger.debug( - 'FEEDS: {}'.format( - json.dumps(self.settings.getdict('FEEDS'), sort_keys=True) - ) - ) - return - yield -""" - Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8") - args = ["-O", "example.json"] - log = self.get_log(spider_code, args=args) - assert ( - '[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}' - in log - ) - with Path(self.cwd, "example.json").open(encoding="utf-8") as f2: - first_line = f2.readline() - assert first_line != "not empty" - - def test_output_and_overwrite_output(self): - spider_code = """ -import scrapy - -class MySpider(scrapy.Spider): - name = 'myspider' - - async def start(self): - return - yield -""" - args = ["-o", "example1.json", "-O", "example2.json"] - log = self.get_log(spider_code, args=args) - assert ( - "error: Please use only one of -o/--output and -O/--overwrite-output" in log - ) - - class TestHelpMessage(TestCommandBase): def setUp(self): super().setUp()