mirror of https://github.com/scrapy/scrapy.git
129 lines
3.6 KiB
Python
129 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
from twisted.web.http import H2_ENABLED
|
|
|
|
from scrapy.utils.reactor import set_asyncio_event_loop_policy
|
|
from tests.keys import generate_keys
|
|
from tests.mockserver.http import MockServer
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Generator
|
|
|
|
|
|
def _py_files(folder):
|
|
return (str(p) for p in Path(folder).rglob("*.py"))
|
|
|
|
|
|
collect_ignore = [
|
|
# may need extra deps
|
|
"docs/_ext",
|
|
# not a test, but looks like a test
|
|
"scrapy/utils/testproc.py",
|
|
"scrapy/utils/testsite.py",
|
|
"tests/ftpserver.py",
|
|
"tests/mockserver.py",
|
|
"tests/pipelines.py",
|
|
"tests/spiders.py",
|
|
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess
|
|
*_py_files("tests/AsyncCrawlerProcess"),
|
|
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess
|
|
*_py_files("tests/AsyncCrawlerRunner"),
|
|
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
|
|
*_py_files("tests/CrawlerProcess"),
|
|
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
|
|
*_py_files("tests/CrawlerRunner"),
|
|
]
|
|
|
|
base_dir = Path(__file__).parent
|
|
ignore_file_path = base_dir / "tests" / "ignores.txt"
|
|
with ignore_file_path.open(encoding="utf-8") as reader:
|
|
for line in reader:
|
|
file_path = line.strip()
|
|
if file_path and file_path[0] != "#":
|
|
collect_ignore.append(file_path)
|
|
|
|
if not H2_ENABLED:
|
|
collect_ignore.extend(
|
|
(
|
|
"scrapy/core/downloader/handlers/http2.py",
|
|
*_py_files("scrapy/core/http2"),
|
|
)
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def mockserver() -> Generator[MockServer]:
|
|
with MockServer() as mockserver:
|
|
yield mockserver
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def reactor_pytest(request) -> str:
|
|
return request.config.getoption("--reactor")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def only_asyncio(request, reactor_pytest):
|
|
if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
|
|
pytest.skip("This test is only run with --reactor=asyncio")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def only_not_asyncio(request, reactor_pytest):
|
|
if (
|
|
request.node.get_closest_marker("only_not_asyncio")
|
|
and reactor_pytest == "asyncio"
|
|
):
|
|
pytest.skip("This test is only run without --reactor=asyncio")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def requires_uvloop(request):
|
|
if not request.node.get_closest_marker("requires_uvloop"):
|
|
return
|
|
try:
|
|
import uvloop # noqa: PLC0415
|
|
|
|
del uvloop
|
|
except ImportError:
|
|
pytest.skip("uvloop is not installed")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def requires_botocore(request):
|
|
if not request.node.get_closest_marker("requires_botocore"):
|
|
return
|
|
try:
|
|
import botocore # noqa: PLC0415
|
|
|
|
del botocore
|
|
except ImportError:
|
|
pytest.skip("botocore is not installed")
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def requires_boto3(request):
|
|
if not request.node.get_closest_marker("requires_boto3"):
|
|
return
|
|
try:
|
|
import boto3 # noqa: PLC0415
|
|
|
|
del boto3
|
|
except ImportError:
|
|
pytest.skip("boto3 is not installed")
|
|
|
|
|
|
def pytest_configure(config):
|
|
if config.getoption("--reactor") == "asyncio":
|
|
# Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.
|
|
# If we decide to run tests with both, we will need to add a new option and check it here.
|
|
set_asyncio_event_loop_policy()
|
|
|
|
|
|
# Generate localhost certificate files, needed by some tests
|
|
generate_keys()
|