mirror of https://github.com/scrapy/scrapy.git
43 lines
1.5 KiB
Python
43 lines
1.5 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING
|
|
|
|
from scrapy.http import Request, Response
|
|
from scrapy.spidermiddlewares.start import StartSpiderMiddleware
|
|
from scrapy.spiders import Spider
|
|
from scrapy.utils.misc import build_from_crawler
|
|
from scrapy.utils.test import get_crawler
|
|
from tests.utils.decorators import coroutine_test
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import AsyncIterator
|
|
|
|
|
|
class TestMiddleware:
|
|
@coroutine_test
|
|
async def test_async(self):
|
|
crawler = get_crawler(Spider)
|
|
mw = build_from_crawler(StartSpiderMiddleware, crawler)
|
|
|
|
async def start() -> AsyncIterator[Request]:
|
|
yield Request("data:,1")
|
|
yield Request("data:,2", meta={"is_start_request": True})
|
|
yield Request("data:,2", meta={"is_start_request": False})
|
|
yield Request("data:,2", meta={"is_start_request": "foo"})
|
|
|
|
result = [
|
|
request.meta["is_start_request"]
|
|
async for request in mw.process_start(start())
|
|
]
|
|
assert result == [True, True, False, "foo"]
|
|
|
|
def test_spider_output_not_marked(self):
|
|
# Requests from a non-None response (spider output) are not flagged.
|
|
crawler = get_crawler(Spider)
|
|
mw = build_from_crawler(StartSpiderMiddleware, crawler)
|
|
response = Response("data:,")
|
|
request = Request("data:,1")
|
|
out = list(mw.process_spider_output(response, [request]))
|
|
assert out == [request]
|
|
assert "is_start_request" not in request.meta
|