mirror of https://github.com/scrapy/scrapy.git
256 lines
10 KiB
Python
256 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from tests.test_commands import TestProjectBase
|
|
from tests.utils.cmdline import call, proc
|
|
|
|
|
|
def write_recording_editor(editor: Path) -> None:
|
|
"""Create an executable editor script that writes the path it is asked to
|
|
open (its last argument) into the file given as its first argument."""
|
|
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
|
|
editor.chmod(0o755)
|
|
|
|
|
|
def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
|
|
"""Find first pattern occurrence in file"""
|
|
pattern = re.compile(regex)
|
|
with filename.open("r", encoding="utf-8") as f:
|
|
for line in f:
|
|
match = pattern.search(line)
|
|
if match is not None:
|
|
return match
|
|
return None
|
|
|
|
|
|
class TestGenspiderCommand(TestProjectBase):
|
|
def test_arguments(self, proj_path: Path) -> None:
|
|
spider = proj_path / self.project_name / "spiders" / "test_name.py"
|
|
# only pass one argument. spider script shouldn't be created
|
|
assert call("genspider", "test_name", cwd=proj_path) == 2
|
|
assert not spider.exists()
|
|
# pass two arguments <name> <domain>. spider script should be created
|
|
assert call("genspider", "test_name", "test.com", cwd=proj_path) == 0
|
|
assert spider.exists()
|
|
|
|
@pytest.mark.parametrize(
|
|
"tplname",
|
|
[
|
|
"basic",
|
|
"crawl",
|
|
"xmlfeed",
|
|
"csvfeed",
|
|
],
|
|
)
|
|
def test_template(self, tplname: str, proj_path: Path) -> None:
|
|
args = [f"--template={tplname}"] if tplname else []
|
|
spname = "test_spider"
|
|
spmodule = f"{self.project_name}.spiders.{spname}"
|
|
spfile = proj_path / self.project_name / "spiders" / f"{spname}.py"
|
|
_, out, _ = proc("genspider", spname, "test.com", *args, cwd=proj_path)
|
|
assert (
|
|
f"Created spider {spname!r} using template {tplname!r} in module:\n {spmodule}"
|
|
in out
|
|
)
|
|
assert spfile.exists()
|
|
modify_time_before = spfile.stat().st_mtime
|
|
_, out, _ = proc("genspider", spname, "test.com", *args, cwd=proj_path)
|
|
assert f"Spider {spname!r} already exists in module" in out
|
|
modify_time_after = spfile.stat().st_mtime
|
|
assert modify_time_after == modify_time_before
|
|
|
|
def test_list(self, proj_path: Path) -> None:
|
|
assert call("genspider", "--list", cwd=proj_path) == 0
|
|
|
|
def test_dump(self, proj_path: Path) -> None:
|
|
assert call("genspider", "--dump=basic", cwd=proj_path) == 0
|
|
assert call("genspider", "-d", "basic", cwd=proj_path) == 0
|
|
|
|
@pytest.mark.skipif(
|
|
sys.platform == "win32", reason="requires a POSIX shell editor script"
|
|
)
|
|
def test_edit(self, proj_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
spider = proj_path / self.project_name / "spiders" / "example2.py"
|
|
edited = proj_path / "edited.txt"
|
|
editor = proj_path / "fake-editor.sh"
|
|
write_recording_editor(editor)
|
|
# The extra argument exercises shlex-splitting of the EDITOR value.
|
|
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
|
|
|
|
returncode, _, err = proc(
|
|
"genspider", "--edit", "example2", "example2.com", cwd=proj_path
|
|
)
|
|
|
|
assert returncode == 0, err
|
|
assert "ModuleNotFoundError" not in err
|
|
assert spider.exists()
|
|
assert (proj_path / edited.read_text(encoding="utf-8")).resolve() == (
|
|
spider.resolve()
|
|
)
|
|
|
|
def test_same_name_as_project(self, proj_path: Path) -> None:
|
|
assert call("genspider", self.project_name, cwd=proj_path) == 2
|
|
assert not (
|
|
proj_path / self.project_name / "spiders" / f"{self.project_name}.py"
|
|
).exists()
|
|
|
|
@pytest.mark.parametrize("force", [True, False])
|
|
def test_same_filename_as_existing_spider(
|
|
self, force: bool, proj_path: Path
|
|
) -> None:
|
|
file_name = "example"
|
|
file_path = proj_path / self.project_name / "spiders" / f"{file_name}.py"
|
|
assert call("genspider", file_name, "example.com", cwd=proj_path) == 0
|
|
assert file_path.exists()
|
|
|
|
# change name of spider but not its file name
|
|
with file_path.open("r+", encoding="utf-8") as spider_file:
|
|
file_data = spider_file.read()
|
|
file_data = file_data.replace('name = "example"', 'name = "renamed"')
|
|
spider_file.seek(0)
|
|
spider_file.write(file_data)
|
|
spider_file.truncate()
|
|
modify_time_before = file_path.stat().st_mtime
|
|
file_contents_before = file_data
|
|
|
|
if force:
|
|
_, out, _ = proc(
|
|
"genspider", "--force", file_name, "example.com", cwd=proj_path
|
|
)
|
|
assert (
|
|
f"Created spider {file_name!r} using template 'basic' in module" in out
|
|
)
|
|
modify_time_after = file_path.stat().st_mtime
|
|
assert modify_time_after != modify_time_before
|
|
file_contents_after = file_path.read_text(encoding="utf-8")
|
|
assert file_contents_after != file_contents_before
|
|
else:
|
|
_, out, _ = proc("genspider", file_name, "example.com", cwd=proj_path)
|
|
assert f"{file_path.resolve()} already exists" in out
|
|
modify_time_after = file_path.stat().st_mtime
|
|
assert modify_time_after == modify_time_before
|
|
file_contents_after = file_path.read_text(encoding="utf-8")
|
|
assert file_contents_after == file_contents_before
|
|
|
|
@pytest.mark.parametrize(
|
|
("url", "domain"),
|
|
[
|
|
("test.com", "test.com"),
|
|
("https://test.com", "test.com"),
|
|
],
|
|
)
|
|
def test_url(self, url: str, domain: str, proj_path: Path) -> None:
|
|
assert call("genspider", "--force", "test_name", url, cwd=proj_path) == 0
|
|
spider = proj_path / self.project_name / "spiders" / "test_name.py"
|
|
m = find_in_file(spider, r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]")
|
|
assert m is not None
|
|
assert m.group(1) == domain
|
|
m = find_in_file(spider, r"start_urls\s*=\s*\[['\"](.+)['\"]\]")
|
|
assert m is not None
|
|
assert m.group(1) == f"https://{domain}"
|
|
|
|
@pytest.mark.parametrize(
|
|
("url", "expected", "template"),
|
|
[
|
|
# basic
|
|
("https://test.com", "https://test.com", "basic"),
|
|
("http://test.com", "http://test.com", "basic"),
|
|
("http://test.com/other/path", "http://test.com/other/path", "basic"),
|
|
("test.com/other/path", "https://test.com/other/path", "basic"),
|
|
# crawl
|
|
("https://test.com", "https://test.com", "crawl"),
|
|
("http://test.com", "http://test.com", "crawl"),
|
|
("http://test.com/other/path", "http://test.com/other/path", "crawl"),
|
|
("test.com/other/path", "https://test.com/other/path", "crawl"),
|
|
("test.com", "https://test.com", "crawl"),
|
|
# xmlfeed
|
|
("https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
|
|
("http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"),
|
|
("test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
|
|
# csvfeed
|
|
("https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
|
|
("http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"),
|
|
("test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
|
|
],
|
|
)
|
|
def test_template_start_urls(
|
|
self, url: str, expected: str, template: str, proj_path: Path
|
|
) -> None:
|
|
assert (
|
|
call(
|
|
"genspider", "-t", template, "--force", "test_name", url, cwd=proj_path
|
|
)
|
|
== 0
|
|
)
|
|
spider = proj_path / self.project_name / "spiders" / "test_name.py"
|
|
m = find_in_file(spider, r"start_urls\s*=\s*\[['\"](.+)['\"]\]")
|
|
assert m is not None
|
|
assert m.group(1) == expected
|
|
|
|
|
|
class TestGenspiderStandaloneCommand:
|
|
def test_generate_standalone_spider(self, tmp_path: Path) -> None:
|
|
call("genspider", "example", "example.com", cwd=tmp_path)
|
|
assert Path(tmp_path, "example.py").exists()
|
|
|
|
@pytest.mark.skipif(
|
|
sys.platform == "win32", reason="requires a POSIX shell editor script"
|
|
)
|
|
def test_edit(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
spider = tmp_path / "example.py"
|
|
edited = tmp_path / "edited.txt"
|
|
editor = tmp_path / "fake-editor.sh"
|
|
write_recording_editor(editor)
|
|
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
|
|
|
|
returncode, _, err = proc(
|
|
"genspider", "--edit", "example", "example.com", cwd=tmp_path
|
|
)
|
|
|
|
assert returncode == 0, err
|
|
assert spider.exists()
|
|
assert (tmp_path / edited.read_text(encoding="utf-8")).resolve() == (
|
|
spider.resolve()
|
|
)
|
|
|
|
@pytest.mark.parametrize("force", [True, False])
|
|
def test_same_name_as_existing_file(self, force: bool, tmp_path: Path) -> None:
|
|
file_name = "example"
|
|
file_path = Path(tmp_path, file_name + ".py")
|
|
_, out, _ = proc("genspider", file_name, "example.com", cwd=tmp_path)
|
|
assert f"Created spider {file_name!r} using template 'basic' " in out
|
|
assert file_path.exists()
|
|
modify_time_before = file_path.stat().st_mtime
|
|
file_contents_before = file_path.read_text(encoding="utf-8")
|
|
|
|
if force:
|
|
# use different template to ensure contents were changed
|
|
_, out, _ = proc(
|
|
"genspider",
|
|
"--force",
|
|
"-t",
|
|
"crawl",
|
|
file_name,
|
|
"example.com",
|
|
cwd=tmp_path,
|
|
)
|
|
assert f"Created spider {file_name!r} using template 'crawl' " in out
|
|
modify_time_after = file_path.stat().st_mtime
|
|
assert modify_time_after != modify_time_before
|
|
file_contents_after = file_path.read_text(encoding="utf-8")
|
|
assert file_contents_after != file_contents_before
|
|
else:
|
|
_, out, _ = proc("genspider", file_name, "example.com", cwd=tmp_path)
|
|
assert (
|
|
f"{Path(tmp_path, file_name + '.py').resolve()} already exists" in out
|
|
)
|
|
modify_time_after = file_path.stat().st_mtime
|
|
assert modify_time_after == modify_time_before
|
|
file_contents_after = file_path.read_text(encoding="utf-8")
|
|
assert file_contents_after == file_contents_before
|