mirror of https://github.com/scrapy/scrapy.git
Type tests involving items and exports (#7867)
This commit is contained in:
Adrian
GitHub
parent
482a02d30c
commit
a5614544a2
|
|
@ -123,23 +123,11 @@ module = [
|
|||
"tests.test_downloaderslotssettings",
|
||||
"tests.test_dupefilters",
|
||||
"tests.test_engine_loop",
|
||||
"tests.test_exporters",
|
||||
"tests.test_extension_statsmailer",
|
||||
"tests.test_extension_throttle",
|
||||
"tests.test_feedexport",
|
||||
"tests.test_feedexport_postprocess",
|
||||
"tests.test_feedexport_storages",
|
||||
"tests.test_feedexport_uri_params",
|
||||
"tests.test_item",
|
||||
"tests.test_linkextractors",
|
||||
"tests.test_loader",
|
||||
"tests.test_logformatter",
|
||||
"tests.test_mail",
|
||||
"tests.test_pipeline_crawl",
|
||||
"tests.test_pipeline_files",
|
||||
"tests.test_pipeline_images",
|
||||
"tests.test_pipeline_media",
|
||||
"tests.test_pipelines",
|
||||
"tests.test_pqueues",
|
||||
"tests.test_scheduler_base",
|
||||
"tests.test_settings",
|
||||
|
|
|
|||
|
|
@ -149,7 +149,7 @@ class BlockingFeedStorage(ABC):
|
|||
|
||||
return NamedTemporaryFile(prefix="feed-", dir=path)
|
||||
|
||||
def store(self, file: IO[bytes]) -> Deferred[None] | None:
|
||||
def store(self, file: IO[bytes]) -> Deferred[None]:
|
||||
return deferred_from_coro(run_in_thread(self._store_in_thread, file))
|
||||
|
||||
@abstractmethod
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from scrapy.utils.defer import ensure_awaitable
|
|||
from scrapy.utils.python import to_bytes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterator
|
||||
from os import PathLike
|
||||
|
||||
from PIL import Image
|
||||
|
|
@ -180,7 +180,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> Iterable[tuple[str, Image.Image, BytesIO]]:
|
||||
) -> Iterator[tuple[str, Image.Image, BytesIO]]:
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
orig_image = self._Image.open(BytesIO(response.body))
|
||||
transposed_image = self._ImageOps.exif_transpose(orig_image)
|
||||
|
|
|
|||
|
|
@ -27,11 +27,12 @@ class MockFTPServer:
|
|||
(anonymous) and a temporary root path that you can read from the
|
||||
:attr:`path` attribute."""
|
||||
|
||||
proc: Popen[str]
|
||||
port: int
|
||||
path: Path
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.proc: Popen[str] | None = None
|
||||
self.host: str = "127.0.0.1"
|
||||
self.port: int | None = None
|
||||
self.path: Path | None = None
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
self.path = Path(mkdtemp())
|
||||
|
|
@ -63,7 +64,6 @@ class MockFTPServer:
|
|||
traceback: TracebackType | None,
|
||||
) -> None:
|
||||
rmtree(str(self.path))
|
||||
assert self.proc is not None
|
||||
self.proc.kill()
|
||||
self.proc.communicate()
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import marshal
|
|||
import pickle
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
|
@ -63,18 +64,18 @@ class TestBaseItemExporter(ABC):
|
|||
self.ie = self._get_exporter()
|
||||
|
||||
@abstractmethod
|
||||
def _get_exporter(self, **kwargs) -> BaseItemExporter:
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
raise NotImplementedError
|
||||
|
||||
def _check_output(self): # noqa: B027
|
||||
def _check_output(self) -> None: # noqa: B027
|
||||
pass
|
||||
|
||||
def _assert_expected_item(self, exported_dict):
|
||||
def _assert_expected_item(self, exported_dict: dict[str, Any]) -> None:
|
||||
for k, v in exported_dict.items():
|
||||
exported_dict[k] = to_unicode(v)
|
||||
assert self.i == self.item_class(**exported_dict)
|
||||
|
||||
def _get_nonstring_types_item(self):
|
||||
def _get_nonstring_types_item(self) -> dict[str, Any]:
|
||||
return {
|
||||
"boolean": False,
|
||||
"number": 22,
|
||||
|
|
@ -82,7 +83,7 @@ class TestBaseItemExporter(ABC):
|
|||
"float": 3.14,
|
||||
}
|
||||
|
||||
def assertItemExportWorks(self, item):
|
||||
def assertItemExportWorks(self, item: Any) -> None:
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(item)
|
||||
self.ie.finish_exporting()
|
||||
|
|
@ -92,7 +93,7 @@ class TestBaseItemExporter(ABC):
|
|||
del self.ie
|
||||
self._check_output()
|
||||
|
||||
def test_export_item(self):
|
||||
def test_export_item(self) -> None:
|
||||
self.assertItemExportWorks(self.i)
|
||||
|
||||
def test_export_dict_item(self):
|
||||
|
|
@ -142,7 +143,7 @@ class TestBaseItemExporter(ABC):
|
|||
|
||||
|
||||
class TestPythonItemExporter(TestBaseItemExporter):
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
return PythonItemExporter(**kwargs)
|
||||
|
||||
def test_invalid_option(self):
|
||||
|
|
@ -173,6 +174,7 @@ class TestPythonItemExporter(TestBaseItemExporter):
|
|||
"age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}],
|
||||
"name": "Jesus",
|
||||
}
|
||||
assert exported is not None
|
||||
assert isinstance(exported["age"][0], dict)
|
||||
assert isinstance(exported["age"][0]["age"][0], dict)
|
||||
|
||||
|
|
@ -186,6 +188,7 @@ class TestPythonItemExporter(TestBaseItemExporter):
|
|||
"age": [{"age": [{"age": "22", "name": "Joseph"}], "name": "Maria"}],
|
||||
"name": "Jesus",
|
||||
}
|
||||
assert exported is not None
|
||||
assert isinstance(exported["age"][0], dict)
|
||||
assert isinstance(exported["age"][0]["age"][0], dict)
|
||||
|
||||
|
|
@ -202,10 +205,10 @@ class TestPythonItemExporterDataclass(TestPythonItemExporter):
|
|||
|
||||
|
||||
class TestPprintItemExporter(TestBaseItemExporter):
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
return PprintItemExporter(self.output, **kwargs)
|
||||
|
||||
def _check_output(self):
|
||||
def _check_output(self) -> None:
|
||||
self._assert_expected_item(eval(self.output.getvalue()))
|
||||
|
||||
|
||||
|
|
@ -215,10 +218,10 @@ class TestPprintItemExporterDataclass(TestPprintItemExporter):
|
|||
|
||||
|
||||
class TestPickleItemExporter(TestBaseItemExporter):
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
return PickleItemExporter(self.output, **kwargs)
|
||||
|
||||
def _check_output(self):
|
||||
def _check_output(self) -> None:
|
||||
self._assert_expected_item(pickle.loads(self.output.getvalue()))
|
||||
|
||||
def test_export_multiple_items(self):
|
||||
|
|
@ -252,10 +255,10 @@ class TestPickleItemExporterDataclass(TestPickleItemExporter):
|
|||
|
||||
|
||||
class TestMarshalItemExporter(TestBaseItemExporter):
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
return MarshalItemExporter(self.output, **kwargs)
|
||||
|
||||
def _check_output(self):
|
||||
def _check_output(self) -> None:
|
||||
self.output.seek(0)
|
||||
self._assert_expected_item(marshal.load(self.output))
|
||||
|
||||
|
|
@ -279,7 +282,7 @@ class TestMarshalItemExporterDataclass(TestMarshalItemExporter):
|
|||
|
||||
|
||||
class TestCsvItemExporter(TestBaseItemExporter):
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
# We need a fresh instance for each exporter, because
|
||||
# CsvItemExporter.stream.__del__() closes the underlying file
|
||||
# (CsvItemExporter.finish_exporting() calls detach() but not all tests
|
||||
|
|
@ -287,8 +290,10 @@ class TestCsvItemExporter(TestBaseItemExporter):
|
|||
self.output = BytesIO()
|
||||
return CsvItemExporter(self.output, **kwargs)
|
||||
|
||||
def assertCsvEqual(self, first, second, msg=None):
|
||||
def split_csv(csv):
|
||||
def assertCsvEqual(
|
||||
self, first: bytes | str, second: bytes | str, msg: str | None = None
|
||||
) -> None:
|
||||
def split_csv(csv: bytes | str) -> list[list[str]]:
|
||||
return [
|
||||
sorted(re.split(r"(,|\s+)", line))
|
||||
for line in to_unicode(csv).splitlines(True)
|
||||
|
|
@ -296,13 +301,15 @@ class TestCsvItemExporter(TestBaseItemExporter):
|
|||
|
||||
assert split_csv(first) == split_csv(second), msg
|
||||
|
||||
def _check_output(self):
|
||||
def _check_output(self) -> None:
|
||||
self.output.seek(0)
|
||||
self.assertCsvEqual(
|
||||
to_unicode(self.output.read()), "age,name\r\n22,John\xa3\r\n"
|
||||
)
|
||||
|
||||
def assertExportResult(self, item, expected, **kwargs):
|
||||
def assertExportResult(
|
||||
self, item: Any, expected: bytes | str = b"", **kwargs: Any
|
||||
) -> None:
|
||||
fp = BytesIO()
|
||||
ie = CsvItemExporter(fp, **kwargs)
|
||||
ie.start_exporting()
|
||||
|
|
@ -383,7 +390,6 @@ class TestCsvItemExporter(TestBaseItemExporter):
|
|||
with pytest.raises(UnicodeEncodeError):
|
||||
self.assertExportResult(
|
||||
item={"text": "W\u0275\u200brd"},
|
||||
expected=None,
|
||||
encoding="windows-1251",
|
||||
)
|
||||
|
||||
|
|
@ -417,7 +423,7 @@ class TestCsvItemExporterDataclass(TestCsvItemExporter):
|
|||
|
||||
|
||||
class TestXmlItemExporter(TestBaseItemExporter):
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
# We need a fresh instance for each exporter, because
|
||||
# XmlItemExporter.stream.__del__() closes the underlying file
|
||||
# (XmlItemExporter.finish_exporting() calls detach() but not all tests
|
||||
|
|
@ -425,20 +431,22 @@ class TestXmlItemExporter(TestBaseItemExporter):
|
|||
self.output = BytesIO()
|
||||
return XmlItemExporter(self.output, **kwargs)
|
||||
|
||||
def assertXmlEquivalent(self, first, second, msg=None):
|
||||
def xmltuple(elem):
|
||||
def assertXmlEquivalent(
|
||||
self, first: bytes, second: bytes, msg: str | None = None
|
||||
) -> None:
|
||||
def xmltuple(elem: Any) -> list[Any]:
|
||||
children = list(elem.iterchildren())
|
||||
if children:
|
||||
return [(child.tag, sorted(xmltuple(child))) for child in children]
|
||||
return [(elem.tag, [(elem.text, ())])]
|
||||
|
||||
def xmlsplit(xmlcontent):
|
||||
def xmlsplit(xmlcontent: bytes) -> list[Any]:
|
||||
doc = lxml.etree.fromstring(xmlcontent)
|
||||
return xmltuple(doc)
|
||||
|
||||
assert xmlsplit(first) == xmlsplit(second), msg
|
||||
|
||||
def assertExportResult(self, item, expected_value):
|
||||
def assertExportResult(self, item: Any, expected_value: bytes) -> None:
|
||||
fp = BytesIO()
|
||||
ie = XmlItemExporter(fp)
|
||||
ie.start_exporting()
|
||||
|
|
@ -447,7 +455,7 @@ class TestXmlItemExporter(TestBaseItemExporter):
|
|||
del ie # See the first “del self.ie” in this file for context.
|
||||
self.assertXmlEquivalent(fp.getvalue(), expected_value)
|
||||
|
||||
def _check_output(self):
|
||||
def _check_output(self) -> None:
|
||||
expected_value = (
|
||||
b'<?xml version="1.0" encoding="utf-8"?>\n'
|
||||
b"<items><item><age>22</age><name>John\xc2\xa3</name></item></items>"
|
||||
|
|
@ -538,10 +546,10 @@ class TestJsonLinesItemExporter(TestBaseItemExporter):
|
|||
"age": {"name": "Maria", "age": {"name": "Joseph", "age": "22"}},
|
||||
}
|
||||
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
return JsonLinesItemExporter(self.output, **kwargs)
|
||||
|
||||
def _check_output(self):
|
||||
def _check_output(self) -> None:
|
||||
exported = json.loads(to_unicode(self.output.getvalue().strip()))
|
||||
assert exported == ItemAdapter(self.i).asdict()
|
||||
|
||||
|
|
@ -582,14 +590,14 @@ class TestJsonLinesItemExporterDataclass(TestJsonLinesItemExporter):
|
|||
class TestJsonItemExporter(TestJsonLinesItemExporter):
|
||||
_expected_nested = [TestJsonLinesItemExporter._expected_nested]
|
||||
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
return JsonItemExporter(self.output, **kwargs)
|
||||
|
||||
def _check_output(self):
|
||||
def _check_output(self) -> None:
|
||||
exported = json.loads(to_unicode(self.output.getvalue().strip()))
|
||||
assert exported == [ItemAdapter(self.i).asdict()]
|
||||
|
||||
def assertTwoItemsExported(self, item):
|
||||
def assertTwoItemsExported(self, item: Any) -> None:
|
||||
self.ie.start_exporting()
|
||||
self.ie.export_item(item)
|
||||
self.ie.export_item(item)
|
||||
|
|
@ -658,7 +666,7 @@ class TestJsonItemExporter(TestJsonLinesItemExporter):
|
|||
|
||||
|
||||
class TestJsonItemExporterToBytes(TestBaseItemExporter):
|
||||
def _get_exporter(self, **kwargs):
|
||||
def _get_exporter(self, **kwargs: Any) -> BaseItemExporter:
|
||||
kwargs["encoding"] = "latin"
|
||||
return JsonItemExporter(self.output, **kwargs)
|
||||
|
||||
|
|
@ -690,7 +698,9 @@ class TestCustomExporterItem:
|
|||
|
||||
def test_exporter_custom_serializer(self):
|
||||
class CustomItemExporter(BaseItemExporter):
|
||||
def serialize_field(self, field, name, value):
|
||||
def serialize_field(
|
||||
self, field: Mapping[str, Any] | Field, name: str, value: Any
|
||||
) -> Any:
|
||||
if name == "age":
|
||||
return str(int(value) + 1)
|
||||
return super().serialize_field(field, name, value)
|
||||
|
|
|
|||
|
|
@ -100,6 +100,8 @@ class InstrumentedFeedSlot(FeedSlot):
|
|||
"""Instrumented FeedSlot subclass for keeping track of calls to
|
||||
start_exporting and finish_exporting."""
|
||||
|
||||
update_listener: Callable[[str], None]
|
||||
|
||||
def start_exporting(self):
|
||||
self.update_listener("start")
|
||||
super().start_exporting()
|
||||
|
|
@ -109,7 +111,7 @@ class InstrumentedFeedSlot(FeedSlot):
|
|||
super().finish_exporting()
|
||||
|
||||
@classmethod
|
||||
def subscribe__listener(cls, listener):
|
||||
def subscribe__listener(cls, listener: IsExportingListener) -> None:
|
||||
cls.update_listener = listener.update
|
||||
|
||||
|
||||
|
|
@ -119,7 +121,7 @@ class IsExportingListener:
|
|||
finish_exporting and when a call to finish_exporting has been made
|
||||
before a call to start_exporting."""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
self.start_without_finish = False
|
||||
self.finish_without_start = False
|
||||
|
||||
|
|
@ -307,6 +309,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert crawler.stats is not None
|
||||
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
|
||||
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
|
||||
|
||||
|
|
@ -330,6 +333,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
side_effect=store,
|
||||
):
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert crawler.stats is not None
|
||||
assert "feedexport/failed_count/FileFeedStorage" in crawler.stats.get_stats()
|
||||
assert crawler.stats.get_value("feedexport/failed_count/FileFeedStorage") == 1
|
||||
|
||||
|
|
@ -347,6 +351,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert crawler.stats is not None
|
||||
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
|
||||
assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()
|
||||
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
|
||||
|
|
@ -487,7 +492,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
|
||||
@coroutine_test
|
||||
async def test_start_finish_exporting_no_items(self):
|
||||
items = []
|
||||
items: list[Any] = []
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
self._random_temp_filename(): {"format": "json"},
|
||||
|
|
@ -526,7 +531,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
|
||||
@coroutine_test
|
||||
async def test_start_finish_exporting_no_items_exception(self):
|
||||
items = []
|
||||
items: list[Any] = []
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
self._random_temp_filename(): {"format": "json"},
|
||||
|
|
@ -611,7 +616,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
items = [{"foo": "bar"}]
|
||||
header = ["foo"]
|
||||
rows = [{"foo": "bar"}]
|
||||
settings = {"FEED_EXPORT_FIELDS": []}
|
||||
settings: dict[str, Any] = {"FEED_EXPORT_FIELDS": []}
|
||||
await self.assertExportedCsv(items, header, rows)
|
||||
await self.assertExportedJsonLines(items, rows, settings)
|
||||
|
||||
|
|
@ -727,14 +732,14 @@ class TestFeedExport(TestFeedExportBase):
|
|||
def accepts(self, item):
|
||||
return isinstance(item, MyItem)
|
||||
|
||||
class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):
|
||||
class CustomFilter2(ItemFilter):
|
||||
def accepts(self, item):
|
||||
return "foo" in item.fields
|
||||
|
||||
class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):
|
||||
class CustomFilter3(ItemFilter):
|
||||
def accepts(self, item):
|
||||
return (
|
||||
isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1"
|
||||
isinstance(item, tuple(self.item_classes)) and item["foo"] == "bar1" # type: ignore[index]
|
||||
)
|
||||
|
||||
formats = {
|
||||
|
|
@ -834,7 +839,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
}
|
||||
|
||||
for fmt, expected in formats.items():
|
||||
settings = {
|
||||
settings: dict[str, Any] = {
|
||||
"FEEDS": {
|
||||
self._random_temp_filename(): {"format": fmt},
|
||||
},
|
||||
|
|
@ -911,7 +916,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
{"key": "value"},
|
||||
]
|
||||
|
||||
test_cases = [
|
||||
test_cases: list[dict[str, Any]] = [
|
||||
# JSON
|
||||
{
|
||||
"format": "json",
|
||||
|
|
@ -1132,7 +1137,7 @@ class TestFeedExport(TestFeedExportBase):
|
|||
|
||||
expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"
|
||||
expected_without_title_csv = b"FOO,BAR\r\n"
|
||||
test_cases = [
|
||||
test_cases: list[dict[str, Any]] = [
|
||||
# with title
|
||||
{
|
||||
"options": {
|
||||
|
|
@ -1166,6 +1171,9 @@ class TestFeedExport(TestFeedExportBase):
|
|||
@coroutine_test
|
||||
async def test_storage_file_no_postprocessing(self):
|
||||
class Storage:
|
||||
open_file: IO[bytes]
|
||||
store_file: IO[bytes]
|
||||
|
||||
def __init__(self, uri, *, feed_options=None):
|
||||
pass
|
||||
|
||||
|
|
@ -1187,6 +1195,10 @@ class TestFeedExport(TestFeedExportBase):
|
|||
@coroutine_test
|
||||
async def test_storage_file_postprocessing(self):
|
||||
class Storage:
|
||||
open_file: IO[bytes]
|
||||
store_file: IO[bytes]
|
||||
file_was_closed: bool
|
||||
|
||||
def __init__(self, uri, *, feed_options=None):
|
||||
pass
|
||||
|
||||
|
|
@ -1299,7 +1311,7 @@ class TestItemFilter:
|
|||
|
||||
class TestFeedExportInit:
|
||||
def test_unsupported_storage(self):
|
||||
settings = {
|
||||
settings: dict[str, Any] = {
|
||||
"FEEDS": {
|
||||
"unsupported://uri": {},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -74,7 +74,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
|
|||
|
||||
return content
|
||||
|
||||
def get_gzip_compressed(self, data, compresslevel=9, mtime=0, filename=""):
|
||||
def get_gzip_compressed(
|
||||
self,
|
||||
data: bytes,
|
||||
compresslevel: int = 9,
|
||||
mtime: int = 0,
|
||||
filename: str = "",
|
||||
) -> bytes:
|
||||
data_stream = BytesIO()
|
||||
gzipf = gzip.GzipFile(
|
||||
fileobj=data_stream,
|
||||
|
|
@ -539,11 +545,13 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
|
|||
|
||||
data = await self.exported_data(self.items, settings)
|
||||
|
||||
for filename, result in data.items():
|
||||
for filename, data_bytes in data.items():
|
||||
expected: Any
|
||||
result: Any
|
||||
if "pickle" in filename:
|
||||
expected, result = self.items[0], pickle.loads(result)
|
||||
expected, result = self.items[0], pickle.loads(data_bytes)
|
||||
elif "marshal" in filename:
|
||||
expected, result = self.items[0], marshal.loads(result)
|
||||
expected, result = self.items[0], marshal.loads(data_bytes)
|
||||
else:
|
||||
expected = filename_to_expected[filename]
|
||||
expected, result = filename_to_expected[filename], data_bytes
|
||||
assert result == expected
|
||||
|
|
|
|||
|
|
@ -95,27 +95,34 @@ class TestFileFeedStorage:
|
|||
assert storage.path == path
|
||||
|
||||
|
||||
def get_test_spider(settings: dict[str, Any] | None = None) -> scrapy.Spider:
|
||||
class TestSpider(scrapy.Spider):
|
||||
name = "test_spider"
|
||||
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
return TestSpider.from_crawler(crawler)
|
||||
|
||||
|
||||
class TestFTPFeedStorage:
|
||||
def get_test_spider(self, settings=None):
|
||||
class TestSpider(scrapy.Spider):
|
||||
name = "test_spider"
|
||||
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
return TestSpider.from_crawler(crawler)
|
||||
|
||||
async def _store(self, uri, content, feed_options=None, settings=None):
|
||||
async def _store(
|
||||
self,
|
||||
uri: str,
|
||||
content: bytes,
|
||||
feed_options: dict[str, Any] | None = None,
|
||||
settings: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
crawler = get_crawler(settings_dict=settings or {})
|
||||
storage = FTPFeedStorage.from_crawler(
|
||||
crawler,
|
||||
uri,
|
||||
feed_options=feed_options,
|
||||
)
|
||||
spider = self.get_test_spider()
|
||||
spider = get_test_spider()
|
||||
file = storage.open(spider)
|
||||
file.write(content)
|
||||
await maybe_deferred_to_future(storage.store(file))
|
||||
|
||||
def _assert_stored(self, path: Path, content):
|
||||
def _assert_stored(self, path: Path, content: bytes) -> None:
|
||||
assert path.exists()
|
||||
try:
|
||||
assert path.read_bytes() == content
|
||||
|
|
@ -165,7 +172,7 @@ class TestFTPFeedStorage:
|
|||
def test_uri_auth_quote(self):
|
||||
# RFC3986: 3.2.1. User Information
|
||||
pw_quoted = quote(string.punctuation, safe="")
|
||||
st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {})
|
||||
st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path")
|
||||
assert st.password == string.punctuation
|
||||
|
||||
def test_uri_without_hostname(self):
|
||||
|
|
@ -181,24 +188,17 @@ class MyBlockingFeedStorage(BlockingFeedStorage):
|
|||
|
||||
|
||||
class TestBlockingFeedStorage:
|
||||
def get_test_spider(self, settings=None):
|
||||
class TestSpider(scrapy.Spider):
|
||||
name = "test_spider"
|
||||
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
return TestSpider.from_crawler(crawler)
|
||||
|
||||
def test_default_temp_dir(self):
|
||||
b = MyBlockingFeedStorage()
|
||||
|
||||
storage_file = b.open(self.get_test_spider())
|
||||
storage_file = b.open(get_test_spider())
|
||||
storage_dir = Path(storage_file.name).parent
|
||||
assert str(storage_dir) == tempfile.gettempdir()
|
||||
|
||||
def test_temp_file(self, tmp_path):
|
||||
b = MyBlockingFeedStorage()
|
||||
|
||||
spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)})
|
||||
spider = get_test_spider({"FEED_TEMPDIR": str(tmp_path)})
|
||||
storage_file = b.open(spider)
|
||||
storage_dir = Path(storage_file.name).parent
|
||||
assert storage_dir == tmp_path
|
||||
|
|
@ -207,7 +207,7 @@ class TestBlockingFeedStorage:
|
|||
b = MyBlockingFeedStorage()
|
||||
|
||||
invalid_path = tmp_path / "invalid_path"
|
||||
spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})
|
||||
spider = get_test_spider({"FEED_TEMPDIR": str(invalid_path)})
|
||||
|
||||
with pytest.raises(OSError, match="Not a Directory:"):
|
||||
b.open(spider=spider)
|
||||
|
|
@ -311,7 +311,7 @@ class TestS3FeedStorage:
|
|||
assert storage.access_key == "access_key"
|
||||
assert storage.secret_key == "secret_key"
|
||||
assert storage.region_name == region_name
|
||||
assert storage.s3_client._client_config.region_name == region_name
|
||||
assert storage.s3_client._client_config.region_name == region_name # type: ignore[attr-defined]
|
||||
|
||||
def test_from_crawler_without_acl(self):
|
||||
settings = {
|
||||
|
|
@ -353,7 +353,7 @@ class TestS3FeedStorage:
|
|||
)
|
||||
assert storage.access_key == "access_key"
|
||||
assert storage.secret_key == "secret_key"
|
||||
assert storage.s3_client._client_config.region_name == "us-east-1"
|
||||
assert storage.s3_client._client_config.region_name == "us-east-1" # type: ignore[attr-defined]
|
||||
|
||||
def test_from_crawler_with_acl(self):
|
||||
settings = {
|
||||
|
|
@ -394,7 +394,7 @@ class TestS3FeedStorage:
|
|||
assert storage.access_key == "access_key"
|
||||
assert storage.secret_key == "secret_key"
|
||||
assert storage.region_name == region_name
|
||||
assert storage.s3_client._client_config.region_name == region_name
|
||||
assert storage.s3_client._client_config.region_name == region_name # type: ignore[attr-defined]
|
||||
|
||||
def test_init_without_max_pool_connections(self) -> None:
|
||||
storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
|
||||
|
|
@ -497,7 +497,7 @@ class TestGCSFeedStorage:
|
|||
def test_parse_empty_acl(self):
|
||||
pytest.importorskip("google.cloud.storage")
|
||||
|
||||
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
|
||||
settings: dict[str, Any] = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
|
||||
assert storage.acl is None
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -10,16 +11,27 @@ from scrapy.exceptions import ScrapyDeprecationWarning
|
|||
from scrapy.extensions.feedexport import FeedExporter
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
class TestURIParams(ABC):
|
||||
spider_name = "uri_params_spider"
|
||||
deprecated_options = False
|
||||
|
||||
@abstractmethod
|
||||
def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
|
||||
def build_settings(
|
||||
self,
|
||||
uri: str = "file:///tmp/foobar",
|
||||
uri_params: Callable[..., dict[str, Any] | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
def _crawler_feed_exporter(self, settings):
|
||||
def _crawler_feed_exporter(
|
||||
self, settings: dict[str, Any]
|
||||
) -> tuple[Crawler, FeedExporter]:
|
||||
if self.deprecated_options:
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
|
|
@ -29,6 +41,7 @@ class TestURIParams(ABC):
|
|||
else:
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
feed_exporter = crawler.get_extension(FeedExporter)
|
||||
assert feed_exporter is not None
|
||||
return crawler, feed_exporter
|
||||
|
||||
def test_default(self):
|
||||
|
|
@ -116,8 +129,12 @@ class TestURIParams(ABC):
|
|||
class TestURIParamsSetting(TestURIParams):
|
||||
deprecated_options = True
|
||||
|
||||
def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
|
||||
extra_settings = {}
|
||||
def build_settings(
|
||||
self,
|
||||
uri: str = "file:///tmp/foobar",
|
||||
uri_params: Callable[..., dict[str, Any] | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
extra_settings: dict[str, Any] = {}
|
||||
if uri_params:
|
||||
extra_settings["FEED_URI_PARAMS"] = uri_params
|
||||
return {
|
||||
|
|
@ -129,8 +146,12 @@ class TestURIParamsSetting(TestURIParams):
|
|||
class TestURIParamsFeedOption(TestURIParams):
|
||||
deprecated_options = False
|
||||
|
||||
def build_settings(self, uri="file:///tmp/foobar", uri_params=None):
|
||||
options = {
|
||||
def build_settings(
|
||||
self,
|
||||
uri: str = "file:///tmp/foobar",
|
||||
uri_params: Callable[..., dict[str, Any] | None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
options: dict[str, Any] = {
|
||||
"format": "jl",
|
||||
}
|
||||
if uri_params:
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from abc import ABCMeta
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -7,9 +8,6 @@ from scrapy.item import Field, Item, ItemMeta
|
|||
|
||||
|
||||
class TestItem:
|
||||
def assertSortedEqual(self, first, second, msg=None):
|
||||
assert sorted(first) == sorted(second), msg
|
||||
|
||||
def test_simple(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
|
|
@ -98,16 +96,16 @@ class TestItem:
|
|||
|
||||
i = TestItem()
|
||||
with pytest.raises(AttributeError):
|
||||
i.name = "john"
|
||||
i.name = "john" # type: ignore[assignment]
|
||||
|
||||
def test_custom_methods(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
|
||||
def get_name(self):
|
||||
def get_name(self) -> Any:
|
||||
return self["name"]
|
||||
|
||||
def change_name(self, name):
|
||||
def change_name(self, name: str) -> None:
|
||||
self["name"] = name
|
||||
|
||||
i = TestItem()
|
||||
|
|
@ -121,40 +119,40 @@ class TestItem:
|
|||
def test_metaclass(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
keys = Field()
|
||||
values = Field()
|
||||
keys = Field() # type: ignore[assignment]
|
||||
values = Field() # type: ignore[assignment]
|
||||
|
||||
i = TestItem()
|
||||
i["name"] = "John"
|
||||
assert list(i.keys()) == ["name"]
|
||||
assert list(i.values()) == ["John"]
|
||||
assert list(i.keys()) == ["name"] # type: ignore[operator]
|
||||
assert list(i.values()) == ["John"] # type: ignore[operator]
|
||||
|
||||
i["keys"] = "Keys"
|
||||
i["values"] = "Values"
|
||||
self.assertSortedEqual(list(i.keys()), ["keys", "values", "name"])
|
||||
self.assertSortedEqual(list(i.values()), ["Keys", "Values", "John"])
|
||||
assert sorted(i.keys()) == ["keys", "name", "values"] # type: ignore[operator]
|
||||
assert sorted(i.values()) == ["John", "Keys", "Values"] # type: ignore[operator]
|
||||
|
||||
def test_metaclass_with_fields_attribute(self):
|
||||
class TestItem(Item):
|
||||
fields = {"new": Field(default="X")}
|
||||
|
||||
item = TestItem(new="New")
|
||||
self.assertSortedEqual(list(item.keys()), ["new"])
|
||||
self.assertSortedEqual(list(item.values()), ["New"])
|
||||
assert list(item.keys()) == ["new"]
|
||||
assert list(item.values()) == ["New"]
|
||||
|
||||
def test_fields_order(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
keys = Field()
|
||||
values = Field()
|
||||
keys = Field() # type: ignore[assignment]
|
||||
values = Field() # type: ignore[assignment]
|
||||
|
||||
assert list(TestItem.fields) == ["name", "keys", "values"]
|
||||
|
||||
def test_fields_order_inheritance(self):
|
||||
class ParentItem(Item):
|
||||
name = Field()
|
||||
keys = Field()
|
||||
values = Field()
|
||||
keys = Field() # type: ignore[assignment]
|
||||
values = Field() # type: ignore[assignment]
|
||||
|
||||
class TestItem(ParentItem):
|
||||
extra = Field()
|
||||
|
|
@ -169,16 +167,16 @@ class TestItem:
|
|||
def test_metaclass_inheritance(self):
|
||||
class ParentItem(Item):
|
||||
name = Field()
|
||||
keys = Field()
|
||||
values = Field()
|
||||
keys = Field() # type: ignore[assignment]
|
||||
values = Field() # type: ignore[assignment]
|
||||
|
||||
class TestItem(ParentItem):
|
||||
keys = Field()
|
||||
|
||||
i = TestItem()
|
||||
i["keys"] = 3
|
||||
assert list(i.keys()) == ["keys"]
|
||||
assert list(i.values()) == [3]
|
||||
assert list(i.keys()) == ["keys"] # type: ignore[operator]
|
||||
assert list(i.values()) == [3] # type: ignore[operator]
|
||||
|
||||
def test_metaclass_multiple_inheritance_simple(self):
|
||||
class A(Item):
|
||||
|
|
@ -314,7 +312,7 @@ class TestItemMeta:
|
|||
def f(self):
|
||||
# For rationale of this see:
|
||||
# https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
|
||||
return __class__
|
||||
return __class__ # type: ignore[name-defined]
|
||||
|
||||
MyItem()
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ class TestBasicItemLoader:
|
|||
|
||||
|
||||
class InitializationTestMixin:
|
||||
item_class: type | None = None
|
||||
item_class: type
|
||||
|
||||
def test_keep_single_value(self):
|
||||
"""Loaded item should contain values from the initial item"""
|
||||
|
|
@ -311,7 +311,7 @@ class TestSelectortemLoader:
|
|||
def test_init_method_with_base_response(self):
|
||||
"""Selector should be None after initialization"""
|
||||
response = Response("https://scrapy.org")
|
||||
l = ProcessorItemLoader(response=response)
|
||||
l = ProcessorItemLoader(response=response) # type: ignore[arg-type]
|
||||
assert l.selector is None
|
||||
|
||||
def test_init_method_with_response(self):
|
||||
|
|
@ -461,6 +461,7 @@ class TestSubselectorLoader:
|
|||
l = NestedItemLoader(response=self.response)
|
||||
|
||||
nl = l.nested_xpath("//header")
|
||||
assert nl.selector is not None
|
||||
nl.add_xpath("name", "div/text()")
|
||||
nl.add_css("name_div", "#id")
|
||||
nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
|
@ -476,6 +477,7 @@ class TestSubselectorLoader:
|
|||
def test_nested_css(self):
|
||||
l = NestedItemLoader(response=self.response)
|
||||
nl = l.nested_css("header")
|
||||
assert nl.selector is not None
|
||||
nl.add_xpath("name", "div/text()")
|
||||
nl.add_css("name_div", "#id")
|
||||
nl.add_value("name_value", nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
|
|
|||
|
|
@ -23,7 +23,10 @@ if TYPE_CHECKING:
|
|||
class MediaDownloadSpider(SimpleSpider):
|
||||
name = "mediadownload"
|
||||
|
||||
def _process_url(self, url):
|
||||
media_key: str
|
||||
media_urls_key: str
|
||||
|
||||
def _process_url(self, url: str) -> str:
|
||||
return url
|
||||
|
||||
def parse(self, response):
|
||||
|
|
@ -44,14 +47,15 @@ class MediaDownloadSpider(SimpleSpider):
|
|||
class BrokenLinksMediaDownloadSpider(MediaDownloadSpider):
|
||||
name = "brokenmedia"
|
||||
|
||||
def _process_url(self, url):
|
||||
def _process_url(self, url: str) -> str:
|
||||
return url + ".foo"
|
||||
|
||||
|
||||
class RedirectedMediaDownloadSpider(MediaDownloadSpider):
|
||||
name = "redirectedmedia"
|
||||
|
||||
def _process_url(self, url):
|
||||
def _process_url(self, url: str) -> str:
|
||||
assert self.mockserver
|
||||
return add_or_replace_parameter(
|
||||
self.mockserver.url("/redirect-to"), "goto", url
|
||||
)
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from itemadapter import ItemAdapter
|
|||
from twisted.internet.defer import Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.item import Field, Item
|
||||
|
|
@ -75,7 +76,7 @@ class DeferredFSFilesStore(FSFilesStore):
|
|||
"""A simple store with persist_file() returning a deferred."""
|
||||
|
||||
def persist_file(self, path, buf, info, meta=None, headers=None):
|
||||
deferred = Deferred()
|
||||
deferred: Deferred[None] = Deferred()
|
||||
# short-hand super() doesn't work in nested functions
|
||||
parent_persist_file = super().persist_file
|
||||
|
||||
|
|
@ -152,7 +153,7 @@ class TestFilesPipeline:
|
|||
file_path(
|
||||
Request("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
response=Response("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
info=object(),
|
||||
info=object(), # type: ignore[arg-type]
|
||||
)
|
||||
== "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1"
|
||||
)
|
||||
|
|
@ -383,7 +384,7 @@ class TestFilesPipeline:
|
|||
"""
|
||||
|
||||
class CustomFilesPipeline(FilesPipeline):
|
||||
def file_path(self, request, response=None, info=None, item=None):
|
||||
def file_path(self, request, response=None, info=None, item=None) -> str:
|
||||
return f"full/{item.get('path')}"
|
||||
|
||||
file_path = CustomFilesPipeline.from_crawler(
|
||||
|
|
@ -476,7 +477,7 @@ class TestFilesPipeline:
|
|||
item["file_urls"] = bad_type
|
||||
|
||||
with pytest.raises(TypeError, match="file_urls must be a list of URLs"):
|
||||
list(pipeline.get_media_requests(item, None))
|
||||
list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestFilesPipelineFieldsMixin(ABC):
|
||||
|
|
@ -491,10 +492,10 @@ class TestFilesPipelineFieldsMixin(ABC):
|
|||
pipeline = FilesPipeline.from_crawler(
|
||||
get_crawler(None, {"FILES_STORE": tmp_path})
|
||||
)
|
||||
requests = list(pipeline.get_media_requests(item, None))
|
||||
requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]
|
||||
assert requests[0].url == url
|
||||
results = [(True, {"url": url})]
|
||||
item = pipeline.item_completed(results, item, None)
|
||||
item = pipeline.item_completed(results, item, None) # type: ignore[arg-type]
|
||||
files = ItemAdapter(item).get("files")
|
||||
assert files == [results[0][1]]
|
||||
assert isinstance(item, self.item_class)
|
||||
|
|
@ -512,10 +513,10 @@ class TestFilesPipelineFieldsMixin(ABC):
|
|||
},
|
||||
)
|
||||
)
|
||||
requests = list(pipeline.get_media_requests(item, None))
|
||||
requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]
|
||||
assert requests[0].url == url
|
||||
results = [(True, {"url": url})]
|
||||
item = pipeline.item_completed(results, item, None)
|
||||
item = pipeline.item_completed(results, item, None) # type: ignore[arg-type]
|
||||
custom_files = ItemAdapter(item).get("custom_files")
|
||||
assert custom_files == [results[0][1]]
|
||||
assert isinstance(item, self.item_class)
|
||||
|
|
@ -581,8 +582,10 @@ class TestFilesPipelineCustomSettings:
|
|||
("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),
|
||||
}
|
||||
|
||||
def _generate_fake_settings(self, tmp_path, prefix=None):
|
||||
def random_string():
|
||||
def _generate_fake_settings(
|
||||
self, tmp_path: Path, prefix: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
def random_string() -> str:
|
||||
return "".join([chr(random.randint(97, 123)) for _ in range(10)])
|
||||
|
||||
settings = {
|
||||
|
|
@ -599,7 +602,7 @@ class TestFilesPipelineCustomSettings:
|
|||
for k, v in settings.items()
|
||||
}
|
||||
|
||||
def _generate_fake_pipeline(self):
|
||||
def _generate_fake_pipeline(self) -> type[FilesPipeline]:
|
||||
class UserDefinedFilePipeline(FilesPipeline):
|
||||
EXPIRES = 1001
|
||||
FILES_URLS_FIELD = "alfa"
|
||||
|
|
@ -739,14 +742,14 @@ class TestFilesPipelineCustomSettings:
|
|||
|
||||
def test_file_pipeline_using_pathlike_objects(self, tmp_path):
|
||||
class CustomFilesPipelineWithPathLikeDir(FilesPipeline):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
return Path("subdir") / Path(request.url).name
|
||||
def file_path(self, request, response=None, info=None, *, item=None) -> str:
|
||||
return str(Path("subdir") / Path(request.url).name)
|
||||
|
||||
pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler(
|
||||
get_crawler(None, {"FILES_STORE": tmp_path})
|
||||
)
|
||||
request = Request("http://example.com/image01.jpg")
|
||||
assert pipeline.file_path(request) == Path("subdir/image01.jpg")
|
||||
assert pipeline.file_path(request) == str(Path("subdir/image01.jpg"))
|
||||
|
||||
|
||||
class TestFSFilesStore:
|
||||
|
|
@ -1092,7 +1095,7 @@ class TestFTPFileStore:
|
|||
store.port,
|
||||
store.username,
|
||||
store.password,
|
||||
store.USE_ACTIVE_MODE,
|
||||
bool(store.USE_ACTIVE_MODE),
|
||||
)
|
||||
assert data == content
|
||||
|
||||
|
|
@ -1160,7 +1163,7 @@ class TestBuildFromCrawler:
|
|||
_from_crawler_called = False
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
def from_crawler(cls, crawler: Crawler) -> "Pipeline":
|
||||
settings = crawler.settings
|
||||
store_uri = settings["FILES_STORE"]
|
||||
o = cls(store_uri, crawler=crawler)
|
||||
|
|
@ -1179,7 +1182,7 @@ def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store):
|
|||
settings = Settings()
|
||||
settings.clear()
|
||||
settings.set("FILES_STORE", store, priority="cmdline")
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
crawler = get_crawler(settings_dict=dict(settings))
|
||||
|
||||
with pytest.raises(NotConfigured):
|
||||
FilesPipeline.from_crawler(crawler)
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ class TestImagesPipeline:
|
|||
file_path(
|
||||
Request("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
response=Response("http://www.dorma.co.uk/images/product_details/2532"),
|
||||
info=object(),
|
||||
info=DUMMY_SPIDER_INFO,
|
||||
)
|
||||
== "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1.jpg"
|
||||
)
|
||||
|
|
@ -120,7 +120,7 @@ class TestImagesPipeline:
|
|||
Request("file:///tmp/some.name/foo"),
|
||||
name,
|
||||
response=Response("file:///tmp/some.name/foo"),
|
||||
info=object(),
|
||||
info=DUMMY_SPIDER_INFO,
|
||||
)
|
||||
== "thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg"
|
||||
)
|
||||
|
|
@ -133,7 +133,7 @@ class TestImagesPipeline:
|
|||
class CustomImagesPipeline(ImagesPipeline):
|
||||
def thumb_path(
|
||||
self, request, thumb_id, response=None, info=None, item=None
|
||||
):
|
||||
) -> str:
|
||||
return f"thumb/{thumb_id}/{item.get('path')}"
|
||||
|
||||
thumb_path = CustomImagesPipeline.from_crawler(
|
||||
|
|
@ -159,11 +159,23 @@ class TestImagesPipeline:
|
|||
req = Request(url="https://dev.mydeco.com/mydeco.gif")
|
||||
|
||||
with pytest.raises(ImageException):
|
||||
next(self.pipeline.get_images(response=resp1, request=req, info=object()))
|
||||
next(
|
||||
self.pipeline.get_images(
|
||||
response=resp1, request=req, info=DUMMY_SPIDER_INFO
|
||||
)
|
||||
)
|
||||
with pytest.raises(ImageException):
|
||||
next(self.pipeline.get_images(response=resp2, request=req, info=object()))
|
||||
next(
|
||||
self.pipeline.get_images(
|
||||
response=resp2, request=req, info=DUMMY_SPIDER_INFO
|
||||
)
|
||||
)
|
||||
with pytest.raises(ImageException):
|
||||
next(self.pipeline.get_images(response=resp3, request=req, info=object()))
|
||||
next(
|
||||
self.pipeline.get_images(
|
||||
response=resp3, request=req, info=DUMMY_SPIDER_INFO
|
||||
)
|
||||
)
|
||||
|
||||
def test_get_images(self):
|
||||
self.pipeline.min_width = 0
|
||||
|
|
@ -176,7 +188,7 @@ class TestImagesPipeline:
|
|||
req = Request(url="https://dev.mydeco.com/mydeco.gif")
|
||||
|
||||
get_images_gen = self.pipeline.get_images(
|
||||
response=resp, request=req, info=object()
|
||||
response=resp, request=req, info=DUMMY_SPIDER_INFO
|
||||
)
|
||||
|
||||
path, new_im, new_buf = next(get_images_gen)
|
||||
|
|
@ -201,7 +213,7 @@ class TestImagesPipeline:
|
|||
req = Request(url="https://dev.mydeco.com/mydeco.gif")
|
||||
|
||||
get_images_gen = self.pipeline.get_images(
|
||||
response=resp, request=req, info=object()
|
||||
response=resp, request=req, info=DUMMY_SPIDER_INFO
|
||||
)
|
||||
|
||||
path, new_im, _ = next(get_images_gen)
|
||||
|
|
@ -230,7 +242,7 @@ class TestImagesPipeline:
|
|||
def test_convert_image(self):
|
||||
SIZE = (100, 100)
|
||||
# straight forward case: RGB and JPEG
|
||||
COLOUR = (0, 127, 255)
|
||||
COLOUR: tuple[int, ...] = (0, 127, 255)
|
||||
im, buf = _create_image("JPEG", "RGB", SIZE, COLOUR)
|
||||
converted, converted_buf = self.pipeline.convert_image(im, response_body=buf)
|
||||
assert converted.mode == "RGB"
|
||||
|
|
@ -296,7 +308,7 @@ class TestImagesPipeline:
|
|||
item["image_urls"] = bad_type
|
||||
|
||||
with pytest.raises(TypeError, match="image_urls must be a list of URLs"):
|
||||
list(pipeline.get_media_requests(item, None))
|
||||
list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))
|
||||
|
||||
|
||||
class TestImagesPipelineFieldsMixin(ABC):
|
||||
|
|
@ -311,10 +323,10 @@ class TestImagesPipelineFieldsMixin(ABC):
|
|||
pipeline = ImagesPipeline.from_crawler(
|
||||
get_crawler(None, {"IMAGES_STORE": "s3://example/images/"})
|
||||
)
|
||||
requests = list(pipeline.get_media_requests(item, None))
|
||||
requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))
|
||||
assert requests[0].url == url
|
||||
results = [(True, {"url": url})]
|
||||
item = pipeline.item_completed(results, item, None)
|
||||
results: Any = [(True, {"url": url})]
|
||||
item = pipeline.item_completed(results, item, DUMMY_SPIDER_INFO)
|
||||
images = ItemAdapter(item).get("images")
|
||||
assert images == [results[0][1]]
|
||||
assert isinstance(item, self.item_class)
|
||||
|
|
@ -332,10 +344,10 @@ class TestImagesPipelineFieldsMixin(ABC):
|
|||
},
|
||||
)
|
||||
)
|
||||
requests = list(pipeline.get_media_requests(item, None))
|
||||
requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))
|
||||
assert requests[0].url == url
|
||||
results = [(True, {"url": url})]
|
||||
item = pipeline.item_completed(results, item, None)
|
||||
results: Any = [(True, {"url": url})]
|
||||
item = pipeline.item_completed(results, item, DUMMY_SPIDER_INFO)
|
||||
custom_images = ItemAdapter(item).get("custom_images")
|
||||
assert custom_images == [results[0][1]]
|
||||
assert isinstance(item, self.item_class)
|
||||
|
|
@ -410,13 +422,15 @@ class TestImagesPipelineCustomSettings:
|
|||
"IMAGES_RESULT_FIELD": "images",
|
||||
}
|
||||
|
||||
def _generate_fake_settings(self, tmp_path, prefix=None):
|
||||
def _generate_fake_settings(
|
||||
self, tmp_path: Path, prefix: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
:param prefix: string for setting keys
|
||||
:return: dictionary of image pipeline settings
|
||||
"""
|
||||
|
||||
def random_string():
|
||||
def random_string() -> str:
|
||||
return "".join([chr(random.randint(97, 123)) for _ in range(10)])
|
||||
|
||||
settings = {
|
||||
|
|
@ -439,7 +453,7 @@ class TestImagesPipelineCustomSettings:
|
|||
for k, v in settings.items()
|
||||
}
|
||||
|
||||
def _generate_fake_pipeline_subclass(self):
|
||||
def _generate_fake_pipeline_subclass(self) -> type[ImagesPipeline]:
|
||||
"""
|
||||
:return: ImagePipeline class will all uppercase attributes set.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -10,7 +11,12 @@ from scrapy import signals
|
|||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.pipelines.files import FileException
|
||||
from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
|
||||
from scrapy.pipelines.media import (
|
||||
FileInfo,
|
||||
FileInfoOrError,
|
||||
MediaPipeline,
|
||||
_MediaRequestFiltered,
|
||||
)
|
||||
from scrapy.utils.defer import _defer_sleep_async
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.signal import disconnect_all
|
||||
|
|
@ -19,21 +25,48 @@ from scrapy.utils.test import get_crawler
|
|||
from tests.utils.decorators import coroutine_test
|
||||
from tests.utils.media_pipelines import mocked_download_func
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Awaitable
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
class UserDefinedPipeline(MediaPipeline):
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
pass
|
||||
def media_to_download(
|
||||
self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
|
||||
) -> Deferred[FileInfo | None] | None:
|
||||
return None
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
pass
|
||||
def get_media_requests(
|
||||
self, item: Any, info: MediaPipeline.SpiderInfo
|
||||
) -> list[Request]:
|
||||
return []
|
||||
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
return {}
|
||||
def media_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> FileInfo | Awaitable[FileInfo]:
|
||||
return cast("FileInfo", {})
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
def media_failed(
|
||||
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
|
||||
) -> Failure:
|
||||
failure.raiseException()
|
||||
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
def file_path(
|
||||
self,
|
||||
request: Request,
|
||||
response: Response | None = None,
|
||||
info: MediaPipeline.SpiderInfo | None = None,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
|
|
@ -48,8 +81,14 @@ class TestBaseMediaPipeline:
|
|||
self.pipe = self.pipeline_class.from_crawler(crawler)
|
||||
self.pipe.open_spider()
|
||||
self.info = self.pipe.spiderinfo
|
||||
assert crawler.request_fingerprinter is not None
|
||||
self.fingerprint = crawler.request_fingerprinter.fingerprint
|
||||
|
||||
@property
|
||||
def mocked_pipe(self) -> MockedMediaPipeline:
|
||||
assert isinstance(self.pipe, MockedMediaPipeline)
|
||||
return self.pipe
|
||||
|
||||
def teardown_method(self):
|
||||
for name, signal in vars(signals).items():
|
||||
if not name.startswith("_"):
|
||||
|
|
@ -121,11 +160,13 @@ class TestBaseMediaPipeline:
|
|||
# When calling the method that caches the Request's result ...
|
||||
self.pipe._cache_result_and_execute_waiters(failure, fp, info)
|
||||
# ... it should store the Twisted Failure ...
|
||||
assert info.downloaded[fp] == failure
|
||||
downloaded = info.downloaded[fp]
|
||||
assert downloaded == failure
|
||||
# ... encapsulating the original FileException ...
|
||||
assert info.downloaded[fp].value == file_exc
|
||||
assert isinstance(downloaded, Failure)
|
||||
assert downloaded.value == file_exc
|
||||
# ... but it should not store the StopIteration exception on its context
|
||||
context = getattr(info.downloaded[fp].value, "__context__", None)
|
||||
context = getattr(downloaded.value, "__context__", None)
|
||||
assert context is None
|
||||
|
||||
def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
|
|
@ -134,7 +175,7 @@ class TestBaseMediaPipeline:
|
|||
|
||||
# Check that failures are logged by default
|
||||
fail = Failure(Exception())
|
||||
results = [(True, 1), (False, fail)]
|
||||
results: Any = [(True, 1), (False, fail)]
|
||||
|
||||
caplog.clear()
|
||||
new_item = self.pipe.item_completed(results, item, self.info)
|
||||
|
|
@ -158,7 +199,7 @@ class TestBaseMediaPipeline:
|
|||
by item_completed(), as they are not download errors."""
|
||||
item = {"name": "name"}
|
||||
fail = Failure(_MediaRequestFiltered("Filtered offsite request"))
|
||||
results = [(True, 1), (False, fail)]
|
||||
results: Any = [(True, 1), (False, fail)]
|
||||
|
||||
with caplog.at_level(logging.DEBUG):
|
||||
new_item = self.pipe.item_completed(results, item, self.info)
|
||||
|
|
@ -174,29 +215,44 @@ class TestBaseMediaPipeline:
|
|||
|
||||
|
||||
class MockedMediaPipeline(UserDefinedPipeline):
|
||||
def __init__(self, *args, crawler=None, **kwargs):
|
||||
def __init__(self, *args: Any, crawler: Crawler, **kwargs: Any):
|
||||
super().__init__(*args, crawler=crawler, **kwargs)
|
||||
self._mockcalled = []
|
||||
self._mockcalled: list[str] = []
|
||||
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
def media_to_download(
|
||||
self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
|
||||
) -> Deferred[FileInfo | None] | None:
|
||||
self._mockcalled.append("media_to_download")
|
||||
if "result" in request.meta:
|
||||
return request.meta.get("result")
|
||||
return super().media_to_download(request, info)
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
def get_media_requests(
|
||||
self, item: Any, info: MediaPipeline.SpiderInfo
|
||||
) -> list[Request]:
|
||||
self._mockcalled.append("get_media_requests")
|
||||
return item.get("requests")
|
||||
return item.get("requests") # type: ignore[no-any-return]
|
||||
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
def media_downloaded(
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> FileInfo | Awaitable[FileInfo]:
|
||||
self._mockcalled.append("media_downloaded")
|
||||
return super().media_downloaded(response, request, info)
|
||||
|
||||
def media_failed(self, failure, request, info):
|
||||
def media_failed(
|
||||
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
|
||||
) -> Failure:
|
||||
self._mockcalled.append("media_failed")
|
||||
return super().media_failed(failure, request, info)
|
||||
|
||||
def item_completed(self, results, item, info):
|
||||
def item_completed(
|
||||
self, results: list[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
|
||||
) -> Any:
|
||||
self._mockcalled.append("item_completed")
|
||||
item = super().item_completed(results, item, info)
|
||||
item["results"] = results
|
||||
|
|
@ -204,7 +260,14 @@ class MockedMediaPipeline(UserDefinedPipeline):
|
|||
|
||||
|
||||
class AsyncMediaDownloadedPipeline(MockedMediaPipeline):
|
||||
async def media_downloaded(self, response, request, info, *, item=None):
|
||||
async def media_downloaded( # type: ignore[override]
|
||||
self,
|
||||
response: Response,
|
||||
request: Request,
|
||||
info: MediaPipeline.SpiderInfo,
|
||||
*,
|
||||
item: Any = None,
|
||||
) -> FileInfo | Awaitable[FileInfo]:
|
||||
return super().media_downloaded(response, request, info)
|
||||
|
||||
|
||||
|
|
@ -212,7 +275,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
pipeline_class = MockedMediaPipeline
|
||||
|
||||
def _errback(self, result):
|
||||
self.pipe._mockcalled.append("request_errback")
|
||||
self.mocked_pipe._mockcalled.append("request_errback")
|
||||
return result
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -226,7 +289,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
item = {"requests": req}
|
||||
new_item = await self.pipe.process_item(item)
|
||||
assert new_item["results"] == [(True, {})]
|
||||
assert self.pipe._mockcalled == [
|
||||
assert self.mocked_pipe._mockcalled == [
|
||||
"get_media_requests",
|
||||
"media_to_download",
|
||||
"media_downloaded",
|
||||
|
|
@ -248,7 +311,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
assert new_item["results"][0][0] is False
|
||||
assert isinstance(new_item["results"][0][1], Failure)
|
||||
assert new_item["results"][0][1].value == exc
|
||||
assert self.pipe._mockcalled == [
|
||||
assert self.mocked_pipe._mockcalled == [
|
||||
"get_media_requests",
|
||||
"media_to_download",
|
||||
"media_failed",
|
||||
|
|
@ -270,7 +333,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
assert new_item["results"][1][0] is False
|
||||
assert isinstance(new_item["results"][1][1], Failure)
|
||||
assert new_item["results"][1][1].value == exc
|
||||
m = self.pipe._mockcalled
|
||||
m = self.mocked_pipe._mockcalled
|
||||
# only once
|
||||
assert m[0] == "get_media_requests" # first hook called
|
||||
assert m.count("get_media_requests") == 1
|
||||
|
|
@ -294,7 +357,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
# returns iterable of Requests
|
||||
req1 = Request("http://url1")
|
||||
req2 = Request("http://url2")
|
||||
item = {"requests": iter([req1, req2])}
|
||||
item = {"requests": iter([req1, req2])} # type: ignore[dict-item]
|
||||
new_item = await self.pipe.process_item(item)
|
||||
assert new_item is item
|
||||
assert self.fingerprint(req1) in self.info.downloaded
|
||||
|
|
@ -304,7 +367,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
async def test_results_are_cached_across_multiple_items(self):
|
||||
rsp1 = Response("http://url1")
|
||||
req1 = Request("http://url1", meta={"response": rsp1})
|
||||
item = {"requests": req1}
|
||||
item: dict[str, Any] = {"requests": req1}
|
||||
new_item = await self.pipe.process_item(item)
|
||||
assert new_item is item
|
||||
assert new_item["results"] == [(True, {})]
|
||||
|
|
@ -335,7 +398,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
new_item = await self.pipe.process_item({"requests": req2})
|
||||
assert new_item["results"][0][0] is False
|
||||
assert new_item["results"][0][1].value is exc
|
||||
assert self.pipe._mockcalled.count("media_to_download") == 1
|
||||
assert self.mocked_pipe._mockcalled.count("media_to_download") == 1
|
||||
|
||||
@coroutine_test
|
||||
async def test_cached_failure_calls_errback(self):
|
||||
|
|
@ -347,13 +410,13 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
)
|
||||
|
||||
def errback(failure):
|
||||
self.pipe._mockcalled.append("request_errback")
|
||||
self.mocked_pipe._mockcalled.append("request_errback")
|
||||
return {"recovered": failure.value}
|
||||
|
||||
req = Request("http://url1", errback=errback)
|
||||
new_item = await self.pipe.process_item({"requests": req})
|
||||
assert new_item["results"] == [(True, {"recovered": exc})]
|
||||
assert self.pipe._mockcalled.count("request_errback") == 1
|
||||
assert self.mocked_pipe._mockcalled.count("request_errback") == 1
|
||||
|
||||
@coroutine_test
|
||||
async def test_results_are_cached_for_requests_of_single_item(self):
|
||||
|
|
@ -362,14 +425,14 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
req2 = Request(
|
||||
req1.url, meta={"response": Response("http://donot.download.me")}
|
||||
)
|
||||
item = {"requests": [req1, req2]}
|
||||
item: dict[str, Any] = {"requests": [req1, req2]}
|
||||
new_item = await self.pipe.process_item(item)
|
||||
assert new_item is item
|
||||
assert new_item["results"] == [(True, {}), (True, {})]
|
||||
|
||||
@coroutine_test
|
||||
async def test_wait_if_request_is_downloading(self):
|
||||
def _check_downloading(response):
|
||||
def _check_downloading(response: Response) -> Response:
|
||||
fp = self.fingerprint(req1)
|
||||
assert fp in self.info.downloading
|
||||
assert fp in self.info.waiting
|
||||
|
|
@ -398,7 +461,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
|
|||
item = {"requests": req}
|
||||
new_item = await self.pipe.process_item(item)
|
||||
assert new_item["results"] == [(True, "ITSME")]
|
||||
assert self.pipe._mockcalled == [
|
||||
assert self.mocked_pipe._mockcalled == [
|
||||
"get_media_requests",
|
||||
"media_to_download",
|
||||
"item_completed",
|
||||
|
|
@ -422,7 +485,9 @@ class TestAsyncMediaDownloaded(TestMediaPipeline):
|
|||
|
||||
|
||||
class TestMediaPipelineAllowRedirectSettings:
|
||||
def _assert_request_no3xx(self, pipeline_class, settings):
|
||||
def _assert_request_no3xx(
|
||||
self, pipeline_class: type[MediaPipeline], settings: dict[str, Any]
|
||||
) -> None:
|
||||
pipe = pipeline_class(crawler=get_crawler(None, settings))
|
||||
request = Request("http://url")
|
||||
pipe._modify_media_request(request)
|
||||
|
|
@ -477,7 +542,7 @@ class TestBuildFromCrawler:
|
|||
self._init_called = True
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
def from_crawler(cls, crawler: Crawler) -> Pipeline:
|
||||
settings = crawler.settings
|
||||
store_uri = settings["FILES_STORE"]
|
||||
o = cls(store_uri, settings=settings, crawler=crawler)
|
||||
|
|
@ -493,9 +558,10 @@ class TestBuildFromCrawler:
|
|||
def test_has_from_crawler(self):
|
||||
class Pipeline(UserDefinedPipeline):
|
||||
_from_crawler_called = False
|
||||
store_uri: str
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
def from_crawler(cls, crawler: Crawler) -> Pipeline:
|
||||
settings = crawler.settings
|
||||
o = super().from_crawler(crawler)
|
||||
o._from_crawler_called = True
|
||||
|
|
@ -509,7 +575,9 @@ class TestBuildFromCrawler:
|
|||
|
||||
|
||||
class MediaFailedNonePipeline(MockedMediaPipeline):
|
||||
def media_failed(self, failure, request, info):
|
||||
def media_failed( # type: ignore[override]
|
||||
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
|
||||
) -> None:
|
||||
self._mockcalled.append("media_failed")
|
||||
|
||||
|
||||
|
|
@ -524,7 +592,7 @@ class TestMediaFailedNone(TestBaseMediaPipeline):
|
|||
req = Request("http://url1", meta={"response": Exception("foo")})
|
||||
new_item = await self.pipe.process_item({"requests": req})
|
||||
assert new_item["results"] == [(True, None)]
|
||||
assert self.pipe._mockcalled == [
|
||||
assert self.mocked_pipe._mockcalled == [
|
||||
"get_media_requests",
|
||||
"media_to_download",
|
||||
"media_failed",
|
||||
|
|
@ -533,7 +601,9 @@ class TestMediaFailedNone(TestBaseMediaPipeline):
|
|||
|
||||
|
||||
class MediaFailedFailurePipeline(MockedMediaPipeline):
|
||||
def media_failed(self, failure, request, info):
|
||||
def media_failed(
|
||||
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
|
||||
) -> Failure:
|
||||
self._mockcalled.append("media_failed")
|
||||
return failure # deprecated
|
||||
|
||||
|
|
@ -544,7 +614,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline):
|
|||
pipeline_class = MediaFailedFailurePipeline
|
||||
|
||||
def _errback(self, result):
|
||||
self.pipe._mockcalled.append("request_errback")
|
||||
self.mocked_pipe._mockcalled.append("request_errback")
|
||||
return result
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -565,7 +635,7 @@ class TestMediaFailedFailure(TestBaseMediaPipeline):
|
|||
assert new_item["results"][0][0] is False
|
||||
assert isinstance(new_item["results"][0][1], Failure)
|
||||
assert new_item["results"][0][1].value == exc
|
||||
assert self.pipe._mockcalled == [
|
||||
assert self.mocked_pipe._mockcalled == [
|
||||
"get_media_requests",
|
||||
"media_to_download",
|
||||
"media_failed",
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class DeferredPipeline:
|
|||
return succeed(None)
|
||||
|
||||
def process_item(self, item):
|
||||
d = Deferred()
|
||||
d: Deferred[Any] = Deferred()
|
||||
d.addCallback(self.cb)
|
||||
d.callback(item)
|
||||
return d
|
||||
|
|
@ -55,7 +55,7 @@ class DeferredPipeline:
|
|||
|
||||
class AsyncDefPipeline:
|
||||
async def process_item(self, item):
|
||||
d = Deferred()
|
||||
d: Deferred[Any] = Deferred()
|
||||
call_later(0, d.callback, None)
|
||||
await maybe_deferred_to_future(d)
|
||||
item["pipeline_passed"] = True
|
||||
|
|
@ -64,7 +64,7 @@ class AsyncDefPipeline:
|
|||
|
||||
class AsyncDefAsyncioPipeline:
|
||||
async def process_item(self, item):
|
||||
d = Deferred()
|
||||
d: Deferred[Any] = Deferred()
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.call_later(0, d.callback, None)
|
||||
await deferred_to_future(d)
|
||||
|
|
@ -75,12 +75,12 @@ class AsyncDefAsyncioPipeline:
|
|||
|
||||
class AsyncDefNotAsyncioPipeline:
|
||||
async def process_item(self, item):
|
||||
d1 = Deferred()
|
||||
d1: Deferred[Any] = Deferred()
|
||||
from twisted.internet import reactor
|
||||
|
||||
reactor.callLater(0, d1.callback, None)
|
||||
await d1
|
||||
d2 = Deferred()
|
||||
d2: Deferred[Any] = Deferred()
|
||||
reactor.callLater(0, d2.callback, None)
|
||||
await maybe_deferred_to_future(d2)
|
||||
item["pipeline_passed"] = True
|
||||
|
|
@ -120,6 +120,8 @@ class OpenSpiderExceptionAsyncPipeline:
|
|||
class ItemSpider(Spider):
|
||||
name = "itemspider"
|
||||
|
||||
mockserver: MockServer
|
||||
|
||||
async def start(self):
|
||||
yield Request(self.mockserver.url("/status?n=200"))
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue