mirror of https://github.com/scrapy/scrapy.git
65 lines
1.8 KiB
Python
65 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
from logging import INFO
|
|
from typing import TYPE_CHECKING
|
|
|
|
import pytest
|
|
|
|
from scrapy.exceptions import NotConfigured
|
|
from scrapy.http import Request, Response
|
|
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
|
|
from scrapy.spiders import Spider
|
|
from scrapy.utils.misc import build_from_crawler
|
|
from scrapy.utils.test import get_crawler
|
|
|
|
if TYPE_CHECKING:
|
|
from scrapy.crawler import Crawler
|
|
from scrapy.statscollectors import StatsCollector
|
|
|
|
|
|
maxlength = 25
|
|
response = Response("http://scrapytest.org")
|
|
short_url_req = Request("http://scrapytest.org/")
|
|
long_url_req = Request("http://scrapytest.org/this_is_a_long_url")
|
|
reqs: list[Request] = [short_url_req, long_url_req]
|
|
|
|
|
|
@pytest.fixture
|
|
def crawler() -> Crawler:
|
|
return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength})
|
|
|
|
|
|
@pytest.fixture
|
|
def stats(crawler: Crawler) -> StatsCollector:
|
|
assert crawler.stats is not None
|
|
return crawler.stats
|
|
|
|
|
|
@pytest.fixture
|
|
def mw(crawler: Crawler) -> UrlLengthMiddleware:
|
|
return UrlLengthMiddleware.from_crawler(crawler)
|
|
|
|
|
|
def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:
|
|
return list(mw.process_spider_output(response, reqs))
|
|
|
|
|
|
def test_middleware_works(mw: UrlLengthMiddleware) -> None:
|
|
assert process_spider_output(mw) == [short_url_req]
|
|
|
|
|
|
def test_not_configured_without_limit() -> None:
|
|
crawler = get_crawler(Spider, {"URLLENGTH_LIMIT": 0})
|
|
with pytest.raises(NotConfigured):
|
|
build_from_crawler(UrlLengthMiddleware, crawler)
|
|
|
|
|
|
def test_logging(
|
|
stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
|
|
) -> None:
|
|
with caplog.at_level(INFO):
|
|
process_spider_output(mw)
|
|
ric = stats.get_value("urllength/request_ignored_count")
|
|
assert ric == 1
|
|
assert f"Ignoring link (url length > {maxlength})" in caplog.text
|