diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 934eb19ac..fcfe7fd29 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -564,6 +564,10 @@ defines the methods described below.
Return response if present in cache, or ``None`` otherwise.
+ If this method raises an exception, e.g. because the cache entry is
+ corrupted, the middleware logs a warning and handles the request as a
+ cache miss.
+
:param spider: the spider which generated the request
:type spider: :class:`~scrapy.Spider` object
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 23738c98c..dfa1e21f6 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -533,8 +533,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
directly
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
- services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
- super proxy that you can attach your own proxies to.
+ services like `ProxyMesh`_.
* for HTTPS websites, if blocking appears related to TLS behavior, consider
adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
@@ -559,5 +558,4 @@ projects that detects common mistakes and anti-patterns.
.. _ProxyMesh: https://proxymesh.com/
.. _Common Crawl: https://commoncrawl.org/
.. _testspiders: https://github.com/scrapinghub/testspiders
-.. _scrapoxy: https://scrapoxy.io/
.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b83a04032..8e565907f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -260,6 +260,8 @@ Request objects
.. automethod:: from_curl
+ .. automethod:: to_curl
+
.. automethod:: to_dict
diff --git a/pyproject.toml b/pyproject.toml
index 6254150f9..0548f6e9b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -178,13 +178,6 @@ module = [
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
- "tests.test_utils_datatypes",
- "tests.test_utils_decorators",
- "tests.test_utils_defer",
- "tests.test_utils_deprecate",
- "tests.test_utils_misc.test_return_with_argument_inside_generator",
- "tests.test_utils_python",
- "tests.test_utils_request",
"tests.utils.bases.http_request",
"tests.utils.bases.http_response",
"tests.utils.bases.spider",
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 6c306afdb..e6d5ff96a 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -225,13 +225,11 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace)
def _run_command_profiled(
cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace
) -> None:
- if opts.profile:
- sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
+ sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
loc = locals()
p = cProfile.Profile()
p.runctx("cmd.run(args, opts)", globals(), loc)
- if opts.profile:
- p.dump_stats(opts.profile)
+ p.dump_stats(opts.profile)
if __name__ == "__main__":
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2ac65bf3f..93194ded7 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand):
self.first_response = response
cb = self._get_callback(spider=spider, opts=opts, response=response)
+ assert response.request
+ response.request.callback = cb
# parse items and requests
depth: int = response.meta["_depth"]
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 07ff4a74e..29b3e3c0f 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler):
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
body = protocol.filename or protocol.body.read()
respcls = responsetypes.from_args(url=request.url, body=body)
- # hints for Headers-related types may need to be fixed to not use AnyStr
- return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]
+ return respcls(url=request.url, status=200, body=body, headers=headers)
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index c6c811809..e7ca0ac0e 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import logging
from email.utils import formatdate
from typing import TYPE_CHECKING
@@ -28,6 +29,9 @@ if TYPE_CHECKING:
from scrapy.statscollectors import StatsCollector
+logger = logging.getLogger(__name__)
+
+
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (
ConnectionDone,
@@ -77,9 +81,20 @@ class HttpCacheMiddleware:
return None
# Look for cached response and check if expired
- cachedresponse: Response | None = self.storage.retrieve_response(
- self.crawler.spider, request
- )
+ cachedresponse: Response | None
+ try:
+ cachedresponse = self.storage.retrieve_response(
+ self.crawler.spider, request
+ )
+ except Exception:
+ self.stats.inc_value("httpcache/retrieve_error")
+ logger.warning(
+ f"Could not read the cache entry for {request}, treating it as a "
+ f"cache miss.",
+ exc_info=True,
+ extra={"spider": self.crawler.spider},
+ )
+ cachedresponse = None
if cachedresponse is None:
self.stats.inc_value("httpcache/miss")
if self.ignore_missing:
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c198677d0..ea38df239 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -13,7 +13,7 @@ import re
import sys
import warnings
from abc import ABC, abstractmethod
-from collections.abc import Callable, Coroutine
+from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
from tempfile import NamedTemporaryFile
@@ -501,7 +501,7 @@ class FeedExporter:
self.feeds = {}
self.slots: list[FeedSlot] = []
self.filters: dict[str, ItemFilter] = {}
- self._pending_close_coros: list[Coroutine[Any, Any, None]] = []
+ self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = []
if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
raise NotConfigured
@@ -565,23 +565,44 @@ class FeedExporter:
)
async def close_spider(self, spider: Spider) -> None:
- self._pending_close_coros.extend(
- self._close_slot(slot, spider) for slot in self.slots
- )
+ for slot in self.slots:
+ self._schedule_slot_close(slot, spider)
- if self._pending_close_coros:
+ if self._pending_close_tasks:
if is_asyncio_available():
await asyncio.wait(
- [asyncio.create_task(coro) for coro in self._pending_close_coros]
+ cast("list[asyncio.Task[None]]", list(self._pending_close_tasks))
)
else:
await DeferredList(
- deferred_from_coro(coro) for coro in self._pending_close_coros
+ cast("list[Deferred[None]]", list(self._pending_close_tasks))
)
# Send FEED_EXPORTER_CLOSED signal
await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
+ def _schedule_slot_close(
+ self, slot: FeedSlot, spider: Spider
+ ) -> asyncio.Task[None] | Deferred[None]:
+ """Start closing the slot without waiting for it to finish, keeping
+ track of the pending work so that it can be awaited in
+ :meth:`close_spider` if it hasn't finished by then."""
+ aw: asyncio.Task[None] | Deferred[None]
+ coro = self._close_slot(slot, spider)
+ if is_asyncio_available():
+ aw = asyncio.create_task(coro)
+ self._pending_close_tasks.append(aw)
+ aw.add_done_callback(self._pending_close_tasks.remove)
+ else:
+ aw = deferred_from_coro(coro)
+ self._pending_close_tasks.append(aw)
+ aw.addBoth(self._untrack_pending_close_task, aw)
+ return aw
+
+ def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any:
+ self._pending_close_tasks.remove(aw)
+ return result
+
@staticmethod
def _get_file(slot_: FeedSlot) -> IO[bytes]:
assert slot_.file
@@ -678,7 +699,7 @@ class FeedExporter:
uri_params = self._get_uri_params(
spider, self.feeds[slot.uri_template]["uri_params"], slot
)
- self._pending_close_coros.append(self._close_slot(slot, spider))
+ self._schedule_slot_close(slot, spider)
slots.append(
self._start_new_batch(
batch_id=slot.batch_id + 1,
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 34d4ec6f2..b55ef6191 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,7 +1,7 @@
from __future__ import annotations
from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
from w3lib.http import headers_dict_to_raw
@@ -25,14 +25,20 @@ class Headers(CaselessDict):
def __init__(
self,
- seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
encoding: str = "utf-8",
):
self.encoding: str = encoding
super().__init__(seq)
def update( # type: ignore[override]
- self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]
+ self,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]],
) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq: dict[bytes, list[bytes]] = {}
@@ -40,7 +46,7 @@ class Headers(CaselessDict):
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
super().update(iseq)
- def normkey(self, key: AnyStr) -> bytes: # type: ignore[override]
+ def normkey(self, key: str | bytes) -> bytes:
"""Normalize key to bytes"""
return self._tobytes(key.title())
@@ -67,19 +73,19 @@ class Headers(CaselessDict):
return str(x).encode(self.encoding)
raise TypeError(f"Unsupported value type: {type(x)}")
- def __getitem__(self, key: AnyStr) -> bytes | None:
+ def __getitem__(self, key: str | bytes) -> bytes | None:
try:
return cast("list[bytes]", super().__getitem__(key))[-1]
except IndexError:
return None
- def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
+ def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
try:
return cast("list[bytes]", super().get(key, def_val))[-1]
except IndexError:
return None
- def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
+ def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
try:
return cast("list[bytes]", super().__getitem__(key))
except KeyError:
@@ -87,15 +93,15 @@ class Headers(CaselessDict):
return self.normvalue(def_val)
return []
- def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
+ def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
self[key] = list_
def setlistdefault(
- self, key: AnyStr, default_list: Iterable[_RawValue] = ()
+ self, key: str | bytes, default_list: Iterable[_RawValue] = ()
) -> Any:
return self.setdefault(key, default_list)
- def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
+ def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
lst = self.getlist(key)
lst.extend(self.normvalue(value))
self[key] = lst
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 73c2e7dd4..68847283a 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -11,7 +11,6 @@ import inspect
from typing import (
TYPE_CHECKING,
Any,
- AnyStr,
Concatenate,
NoReturn,
TypeAlias,
@@ -125,7 +124,10 @@ class Request(object_ref):
url: str,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@@ -310,7 +312,11 @@ class Request(object_ref):
@headers.setter
def headers(
- self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+ self,
+ value: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None,
) -> None:
if isinstance(value, Headers):
self._headers = value
@@ -381,6 +387,20 @@ class Request(object_ref):
request_kwargs.update(kwargs)
return cls(**request_kwargs)
+ def to_curl(self) -> str:
+ """Return a string with a `cURL `_ command equivalent
+ to this request.
+
+ Inverse of :meth:`from_curl`. See also
+ :func:`scrapy.utils.request.request_to_curl`.
+
+ .. versionadded:: VERSION
+ """
+ # Imported here to avoid a circular import.
+ from scrapy.utils.request import request_to_curl # noqa: PLC0415
+
+ return request_to_curl(self)
+
def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
"""Return a dictionary containing the Request's data.
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 09b1c8b32..f1db11488 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload
+from typing import TYPE_CHECKING, Any, TypeVar, overload
from urllib.parse import urljoin
from scrapy.exceptions import NotSupported
@@ -72,7 +72,10 @@ class Response(object_ref):
self,
url: str,
status: int = 200,
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes = b"",
flags: list[str] | None = None,
request: Request | None = None,
@@ -145,7 +148,11 @@ class Response(object_ref):
@headers.setter
def headers(
- self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+ self,
+ value: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None,
) -> None:
if isinstance(value, Headers):
self._headers = value
@@ -222,7 +229,10 @@ class Response(object_ref):
url: str | Link,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@@ -272,7 +282,10 @@ class Response(object_ref):
urls: Iterable[str | Link],
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 6876e35e8..d01e23e47 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -9,7 +9,7 @@ from __future__ import annotations
import json
from contextlib import suppress
-from typing import TYPE_CHECKING, Any, AnyStr, cast
+from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urljoin
import parsel
@@ -170,7 +170,10 @@ class TextResponse(Response):
url: str | Link | parsel.Selector,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
@@ -223,7 +226,10 @@ class TextResponse(Response):
urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
callback: CallbackT | None = None,
method: str = "GET",
- headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ headers: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
body: bytes | str | None = None,
cookies: CookiesT | None = None,
meta: dict[str, Any] | None = None,
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 97123e63c..0691312a3 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -2,6 +2,8 @@
Mail sending helpers
"""
+# pragma: no file cover
+
from __future__ import annotations
import logging
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 925f31ede..1e7ac9c34 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -9,7 +9,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any
-from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.exceptions import NotSupported
from scrapy.http import Response, TextResponse
from scrapy.selector import Selector
from scrapy.spiders import Spider
@@ -76,11 +76,6 @@ class XMLFeedSpider(Spider):
yield from self.process_results(response, ret)
def _parse(self, response: Response, **kwargs: Any) -> Any:
- if not hasattr(self, "parse_node"):
- raise NotConfigured(
- "You must define parse_node method in order to scrape this XML feed"
- )
-
response = self.adapt_response(response)
nodes: Iterable[Selector]
if self.iterator == "iternodes":
@@ -158,9 +153,5 @@ class CSVFeedSpider(Spider):
yield from self.process_results(response, ret)
def _parse(self, response: Response, **kwargs: Any) -> Any:
- if not hasattr(self, "parse_row"):
- raise NotConfigured(
- "You must define parse_row method in order to scrape this CSV feed"
- )
response = self.adapt_response(response)
return self.parse_rows(response)
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index c020ff4b9..9a945c61c 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -11,12 +11,12 @@ import warnings
import weakref
from collections import OrderedDict
from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeVar, cast
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
- from collections.abc import Iterable, Sequence
+ from collections.abc import Container, Iterable
# typing.Self requires Python 3.11
from typing_extensions import Self
@@ -44,22 +44,25 @@ class CaselessDict(dict): # type: ignore[type-arg]
def __init__(
self,
- seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]]
+ | None = None,
):
super().__init__()
if seq:
self.update(seq)
- def __getitem__(self, key: AnyStr) -> Any:
+ def __getitem__(self, key: str | bytes) -> Any:
return dict.__getitem__(self, self.normkey(key))
- def __setitem__(self, key: AnyStr, value: Any) -> None:
+ def __setitem__(self, key: str | bytes, value: Any) -> None:
dict.__setitem__(self, self.normkey(key), self.normvalue(value))
- def __delitem__(self, key: AnyStr) -> None:
+ def __delitem__(self, key: str | bytes) -> None:
dict.__delitem__(self, self.normkey(key))
- def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
+ def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
return dict.__contains__(self, self.normkey(key))
has_key = __contains__
@@ -69,7 +72,7 @@ class CaselessDict(dict): # type: ignore[type-arg]
copy = __copy__
- def normkey(self, key: AnyStr) -> AnyStr:
+ def normkey(self, key: str | bytes) -> str | bytes:
"""Method to normalize dictionary key access"""
return key.lower()
@@ -77,23 +80,28 @@ class CaselessDict(dict): # type: ignore[type-arg]
"""Method to normalize values prior to be set"""
return value
- def get(self, key: AnyStr, def_val: Any = None) -> Any:
+ def get(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.get(self, self.normkey(key), self.normvalue(def_val))
- def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:
+ def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
# doesn't fully implement MutableMapping.update()
- def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None: # type: ignore[override]
+ def update( # type: ignore[override]
+ self,
+ seq: Mapping[str, Any]
+ | Mapping[bytes, Any]
+ | Iterable[tuple[str | bytes, Any]],
+ ) -> None:
seq = seq.items() if isinstance(seq, Mapping) else seq
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
super().update(iseq)
@classmethod
- def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self: # type: ignore[override]
- return cls((k, value) for k in keys) # type: ignore[misc]
+ def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self: # type: ignore[override]
+ return cls((k, value) for k in keys)
- def pop(self, key: AnyStr, *args: Any) -> Any:
+ def pop(self, key: str | bytes, *args: Any) -> Any:
return dict.pop(self, self.normkey(key), *args)
@@ -132,6 +140,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
def __repr__(self) -> str:
return f"<{self.__class__.__name__}: {super().__repr__()}>"
+ # UserDict.copy() shallow-copies the instance, which would share self._keys
+ # between the copy and the original.
+ def __copy__(self) -> Self:
+ new = self.__class__()
+ new.data = self.data.copy()
+ new._keys = self._keys.copy()
+ return new
+
+ copy = __copy__
+
+ # UserDict.__ior__ updates self.data directly, which would leave self._keys
+ # out of date.
+ def __ior__(self, other: Any) -> Self: # type: ignore[override,misc]
+ self.update(other)
+ return self
+
def _normkey(self, key: str | bytes) -> str | bytes:
return key
@@ -189,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
class SequenceExclude:
"""Object to test if an item is NOT within some sequence."""
- def __init__(self, seq: Sequence[Any]):
- self.seq: Sequence[Any] = seq
+ def __init__(self, seq: Container[Any]):
+ self.seq: Container[Any] = seq
def __contains__(self, item: Any) -> bool:
return item not in self.seq
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index a5bb6fa24..4960dc27a 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -19,9 +19,19 @@ _T = TypeVar("_T")
_P = ParamSpec("_P")
+@overload
+def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ...
+
+
+@overload
def deprecated(
- use_instead: Any = None,
-) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+ use_instead: str | None = None,
+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...
+
+
+def deprecated(
+ use_instead: Callable[_P, _T] | str | None = None,
+) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:
"""This is a decorator which can be used to mark functions
as deprecated. It will result in a warning being emitted
when the function is used."""
@@ -38,8 +48,9 @@ def deprecated(
return wrapped
if callable(use_instead):
- deco = deco(use_instead)
+ func = use_instead
use_instead = None
+ return deco(func)
return deco
diff --git a/tests/spiders.py b/tests/spiders.py
index da14fdbe3..7c7d3007c 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -38,6 +38,35 @@ class MockServerSpider(Spider):
self.is_secure = is_secure
+class RawResponseSpider(MockServerSpider):
+ """Base class for spiders that fetch a response built by the test itself.
+
+ Subclasses return the body from :meth:`raw_body` and request
+ :attr:`raw_url`, which the mock server answers with that body verbatim
+ under :attr:`content_type`. This lets tests reach parsing code that only
+ a specific kind of response triggers while still going through a regular
+ crawl, instead of calling internal parsing methods directly.
+ """
+
+ name = "raw_response"
+ content_type = "text/plain"
+
+ def raw_body(self) -> str:
+ raise NotImplementedError
+
+ @property
+ def raw_url(self) -> str:
+ assert self.mockserver
+ raw = (
+ "HTTP/1.1 200 OK\r\n"
+ f"Content-Type: {self.content_type}\r\n"
+ "Connection: close\r\n"
+ "\r\n"
+ f"{self.raw_body()}"
+ )
+ return self.mockserver.url("/raw?" + urlencode({"raw": raw}))
+
+
class MetaSpider(MockServerSpider):
name = "meta"
@@ -496,6 +525,23 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
self.logger.info("[errback] status %i", failure.value.response.status)
+class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
+ name = "crawl_spider_without_errback"
+
+ async def start(self):
+ test_body = b"""
+
+ Page title
+
+ Item 200
+ Item 404
+
+
+ """
+ url = self.mockserver.url("/alpayload")
+ yield Request(url, method="POST", body=test_body)
+
+
class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod):
name = "crawl_spider_with_process_request_cb_kwargs"
rules = (
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 9b7131c7a..772cc82e2 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -126,6 +126,32 @@ class MySpider(scrapy.Spider):
else:
self.logger.debug('It Does Not Work :(')
+class RetryRequestSpider(BaseSpider):
+ name = 'retry_request'
+
+ def parse(self, response):
+ if response.meta.get('retried'):
+ yield {{'retried': True}}
+ return
+ response.meta['retried'] = True
+ yield response.request.replace(dont_filter=True)
+
+class CustomCallbackRetryRequestSpider(BaseSpider):
+ name = 'retry_request_custom_callback'
+
+ def parse(self, response):
+ yield response.request.replace(
+ callback=self.parse_retry,
+ dont_filter=True,
+ )
+
+ def parse_retry(self, response):
+ if response.meta.get('retried'):
+ yield {{'retried_with_custom_callback': True}}
+ return
+ response.meta['retried'] = True
+ yield response.request.replace(dont_filter=True)
+
class MyGoodCrawlSpider(CrawlSpider):
name = 'goodcrawl{self.spider_name}'
@@ -381,6 +407,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "[{}, {'foo': 'bar'}]" in out
+ def test_retry_response_request(
+ self, proj_path: Path, mockserver: MockServer
+ ) -> None:
+ _, out, stderr = proc(
+ "parse",
+ "--spider",
+ "retry_request",
+ "-d",
+ "2",
+ mockserver.url("/html"),
+ cwd=proj_path,
+ )
+ assert "RecursionError" not in stderr
+ assert "{'retried': True}" in out
+
+ def test_retry_response_request_with_custom_callback(
+ self, proj_path: Path, mockserver: MockServer
+ ) -> None:
+ _, out, stderr = proc(
+ "parse",
+ "--spider",
+ "retry_request_custom_callback",
+ "-d",
+ "3",
+ mockserver.url("/html"),
+ cwd=proj_path,
+ )
+ assert "RecursionError" not in stderr
+ assert "{'retried_with_custom_callback': True}" in out
+
def test_wrong_callback_passed(
self, proj_path: Path, mockserver: MockServer
) -> None:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 51f98db1b..3e687e811 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -3,14 +3,12 @@ from __future__ import annotations
import argparse
import json
import sys
-from io import StringIO
from typing import TYPE_CHECKING
-from unittest import mock
import pytest
import scrapy
-from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg
+from scrapy.cmdline import _pop_command_name, execute
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
@@ -153,12 +151,6 @@ class MySpider(scrapy.Spider):
self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n")
- @staticmethod
- def _append_settings(proj_mod_path: Path, text: str) -> None:
- """Add text to the end of the project settings.py."""
- with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
- f.write(text)
-
@staticmethod
def _replace_custom_settings(
proj_mod_path: Path, spider_name: str, text: str
@@ -347,23 +339,223 @@ class TestMiscCommands(TestProjectBase):
subdir.mkdir(exist_ok=True)
assert call("list", cwd=subdir) == 0
- def test_command_not_found(self) -> None:
- na_msg = """
-The list command is not available from this location.
-These commands are only available from within a project: check, crawl, edit, list, parse.
-"""
- not_found_msg = """
-Unknown command: abc
-"""
- params = [
- ("list", False, na_msg),
- ("abc", False, not_found_msg),
- ("abc", True, not_found_msg),
- ]
- for cmdname, inproject, message in params:
- with mock.patch("sys.stdout", new=StringIO()) as out:
- _print_unknown_command_msg(Settings(), cmdname, inproject)
- assert out.getvalue().strip() == message.strip()
+
+class TestCommandListing(TestProjectBase):
+ """Tests for the command list that ``scrapy`` prints when called without a
+ command name."""
+
+ def test_outside_project(self) -> None:
+ returncode, out, err = proc()
+ assert returncode == 0, err
+ assert f"Scrapy {scrapy.__version__} - no active project" in out
+ assert "Available commands:" in out
+ assert "Create new project" in out
+ assert "More commands available when run from project directory" in out
+ assert 'Use "scrapy -h" to see more info about a command' in out
+
+ def test_inside_project(self, proj_path: Path) -> None:
+ returncode, out, err = proc(cwd=proj_path)
+ assert returncode == 0, err
+ assert (
+ f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
+ )
+ assert "List available spiders" in out
+ assert "More commands available when run from project directory" not in out
+
+
+class TestUnknownCommand(TestProjectBase):
+ def test_outside_project(self) -> None:
+ returncode, out, err = proc("abc")
+ assert returncode == 2, err
+ assert f"Scrapy {scrapy.__version__} - no active project" in out
+ assert "Unknown command: abc" in out
+ assert 'Use "scrapy" to see available commands' in out
+
+ def test_inside_project(self, proj_path: Path) -> None:
+ returncode, out, err = proc("abc", cwd=proj_path)
+ assert returncode == 2, err
+ assert (
+ f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
+ )
+ assert "Unknown command: abc" in out
+
+ def test_project_only_command_outside_project(self) -> None:
+ returncode, out, err = proc("list")
+ assert returncode == 2, err
+ assert "The list command is not available from this location." in out
+ assert (
+ "These commands are only available from within a project: "
+ "check, crawl, edit, list, parse." in out
+ )
+
+
+class TestCommandsModule(TestProjectBase):
+ """Tests for commands defined in the module of the COMMANDS_MODULE setting."""
+
+ @pytest.fixture
+ def proj_path_with_commands(self, proj_path: Path) -> Path:
+ commands_path = proj_path / self.project_name / "commands"
+ commands_path.mkdir()
+ (commands_path / "__init__.py").touch()
+ (commands_path / "mycmd.py").write_text(
+ """
+from scrapy.commands import ScrapyCommand
+
+
+class Command(ScrapyCommand):
+ requires_crawler_process = False
+
+ def short_desc(self):
+ return "My custom command"
+
+ def run(self, args, opts):
+ print("My custom command ran")
+""",
+ encoding="utf-8",
+ )
+ (commands_path / "helpcmd.py").write_text(
+ """
+from scrapy.commands import ScrapyCommand
+from scrapy.exceptions import UsageError
+
+
+class Command(ScrapyCommand):
+ requires_crawler_process = False
+
+ def short_desc(self):
+ return "My command that asks for its help message"
+
+ def run(self, args, opts):
+ raise UsageError
+""",
+ encoding="utf-8",
+ )
+ (commands_path / "silentcmd.py").write_text(
+ """
+from scrapy.commands import ScrapyCommand
+from scrapy.exceptions import UsageError
+
+
+class Command(ScrapyCommand):
+ requires_crawler_process = False
+
+ def short_desc(self):
+ return "My command that fails silently"
+
+ def run(self, args, opts):
+ raise UsageError(print_help=False)
+""",
+ encoding="utf-8",
+ )
+ self._append_settings(
+ proj_path / self.project_name,
+ f'\nCOMMANDS_MODULE = "{self.project_name}.commands"\n',
+ )
+ return proj_path
+
+ def test_listed(self, proj_path_with_commands: Path) -> None:
+ returncode, out, err = proc(cwd=proj_path_with_commands)
+ assert returncode == 0, err
+ assert "My custom command" in out
+
+ def test_run(self, proj_path_with_commands: Path) -> None:
+ returncode, out, err = proc("mycmd", cwd=proj_path_with_commands)
+ assert returncode == 0, err
+ assert "My custom command ran" in out
+
+ def test_usage_error(self, proj_path_with_commands: Path) -> None:
+ """A message-less UsageError makes the help message be printed."""
+ returncode, out, err = proc("helpcmd", cwd=proj_path_with_commands)
+ assert returncode == 2, err
+ assert "scrapy helpcmd" in out
+
+ def test_usage_error_without_help(self, proj_path_with_commands: Path) -> None:
+ """A message-less UsageError with print_help disabled prints nothing."""
+ returncode, out, err = proc("silentcmd", cwd=proj_path_with_commands)
+ assert returncode == 2, err
+ assert not out
+
+
+class TestEntryPointCommands:
+ """Tests for commands defined in the scrapy.commands entry point group."""
+
+ @staticmethod
+ def _write_dist(path: Path, entry_point: str) -> None:
+ """Write into *path* a package with a command and a function, and the
+ metadata of an installed distribution that declares *entry_point* in
+ the scrapy.commands entry point group.
+
+ Since ``python -m scrapy.cmdline`` puts the current working directory
+ in the import path, running it with *path* as the working directory
+ makes Scrapy find that entry point.
+ """
+ package_path = path / "mycmds"
+ package_path.mkdir()
+ (package_path / "__init__.py").touch()
+ (package_path / "mycmd.py").write_text(
+ """
+from scrapy.commands import ScrapyCommand
+
+
+class Command(ScrapyCommand):
+ requires_crawler_process = False
+
+ def short_desc(self):
+ return "My entry point command"
+
+ def run(self, args, opts):
+ print("My entry point command ran")
+
+
+def not_a_command():
+ pass
+""",
+ encoding="utf-8",
+ )
+ dist_info_path = path / "mycmds-1.0.dist-info"
+ dist_info_path.mkdir()
+ (dist_info_path / "METADATA").write_text(
+ "Metadata-Version: 2.1\nName: mycmds\nVersion: 1.0\n", encoding="utf-8"
+ )
+ (dist_info_path / "entry_points.txt").write_text(
+ f"[scrapy.commands]\n{entry_point}\n", encoding="utf-8"
+ )
+
+ def test_listed(self, tmp_path: Path) -> None:
+ self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
+ returncode, out, err = proc(cwd=tmp_path)
+ assert returncode == 0, err
+ assert "My entry point command" in out
+
+ def test_run(self, tmp_path: Path) -> None:
+ self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
+ returncode, out, err = proc("mycmd", cwd=tmp_path)
+ assert returncode == 0, err
+ assert "My entry point command ran" in out
+
+ def test_not_a_class(self, tmp_path: Path) -> None:
+ self._write_dist(tmp_path, "mycmd = mycmds.mycmd:not_a_command")
+ returncode, _, err = proc("version", cwd=tmp_path)
+ assert returncode == 1
+ assert "ValueError: Invalid entry point mycmd" in err
+
+
+class TestExecute:
+ """Tests for calls to scrapy.cmdline.execute() from Python code, which the
+ command line does not cover."""
+
+ def test_argv(self, capsys: pytest.CaptureFixture[str]) -> None:
+ with pytest.raises(SystemExit) as exc_info:
+ execute(["scrapy", "version"])
+ assert exc_info.value.code == 0
+ assert scrapy.__version__ in capsys.readouterr().out
+
+ def test_settings(self, capsys: pytest.CaptureFixture[str]) -> None:
+ settings = Settings()
+ with pytest.raises(SystemExit) as exc_info:
+ execute(["scrapy", "settings", "--get", "BOT_NAME"], settings=settings)
+ assert exc_info.value.code == 0
+ assert capsys.readouterr().out.strip() == "scrapybot"
class TestBenchCommand:
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 9c479068a..d284805be 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -41,6 +41,7 @@ from tests.spiders import (
CrawlSpiderWithAsyncCallback,
CrawlSpiderWithAsyncGeneratorCallback,
CrawlSpiderWithErrback,
+ CrawlSpiderWithoutErrback,
CrawlSpiderWithParseMethod,
CrawlSpiderWithProcessRequestCallbackKeywordArguments,
DelaySpider,
@@ -505,6 +506,21 @@ class TestCrawlSpider:
assert "[errback] status 500" in caplog.text
assert "[errback] status 501" in caplog.text
+ @coroutine_test
+ async def test_crawlspider_without_errback(
+ self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+ ) -> None:
+ crawler = get_crawler(CrawlSpiderWithoutErrback)
+ with caplog.at_level(logging.INFO):
+ await crawler.crawl_async(mockserver=mockserver)
+
+ # The failing request (404) is followed by a rule without an errback,
+ # so the failure is dropped silently and the crawl finishes normally.
+ assert "[parse] status 200 (foo: None)" in caplog.text
+ assert "[errback]" not in caplog.text
+ assert crawler.stats
+ assert crawler.stats.get_value("downloader/response_status_count/404") == 1
+
@coroutine_test
async def test_crawlspider_process_request_cb_kwargs(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9d5d6874e..ce56ee11d 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -1,10 +1,12 @@
from __future__ import annotations
import email.utils
+import logging
import shutil
import tempfile
import time
from contextlib import contextmanager
+from pathlib import Path
from typing import TYPE_CHECKING, Any
from unittest import mock
@@ -93,6 +95,12 @@ class TestBase:
class StorageTestMixin:
"""Mixin containing storage-specific test methods."""
+ def _corrupt_cache_entry(
+ self, storage: Any, spider: Spider, request: Request
+ ) -> None:
+ """Make the cache entry of *request* unreadable for *storage*."""
+ raise NotImplementedError
+
def test_storage(self):
with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler):
request2 = self.request.copy()
@@ -115,6 +123,38 @@ class StorageTestMixin:
with mock.patch("scrapy.extensions.httpcache.time", return_value=future):
assert storage.retrieve_response(crawler.spider, self.request)
+ def test_corrupted_cache_entry_is_a_miss(self, caplog):
+ with self._middleware() as mw:
+ spider = mw.crawler.spider
+ mw.storage.store_response(spider, self.request, self.response)
+ self._corrupt_cache_entry(mw.storage, spider, self.request)
+
+ caplog.clear()
+ with caplog.at_level(logging.WARNING):
+ assert mw.process_request(self.request) is None
+
+ assert "treating it as a cache miss" in caplog.text
+ assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
+ assert mw.crawler.stats.get_value("httpcache/miss") == 1
+
+ # Storing the response again replaces the corrupted cache entry.
+ mw.storage.store_response(spider, self.request, self.response)
+ self.assertEqualResponse(
+ self.response, mw.storage.retrieve_response(spider, self.request)
+ )
+
+ def test_corrupted_cache_entry_ignore_missing(self):
+ with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
+ spider = mw.crawler.spider
+ mw.storage.store_response(spider, self.request, self.response)
+ self._corrupt_cache_entry(mw.storage, spider, self.request)
+
+ with pytest.raises(IgnoreRequest):
+ mw.process_request(self.request)
+
+ assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
+ assert mw.crawler.stats.get_value("httpcache/ignore") == 1
+
def test_storage_no_content_type_header(self):
"""Test that the response body is used to get the right response class
even if there is no Content-Type header"""
@@ -556,29 +596,43 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
# Concrete test classes that combine storage and policy mixins
-class TestFilesystemStorageWithDummyPolicy(
- TestBase, StorageTestMixin, DummyPolicyTestMixin
-):
+class FilesystemStorageTestMixin(StorageTestMixin):
storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+ def _corrupt_cache_entry(self, storage, spider, request) -> None:
+ rpath = Path(storage._get_request_path(spider, request))
+ (rpath / "response_body").unlink()
+
+
+class DbmStorageTestMixin(StorageTestMixin):
+ storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+
+ def _corrupt_cache_entry(self, storage, spider, request) -> None:
+ key = storage._fingerprinter.fingerprint(request).hex()
+ storage.db[f"{key}_data"] = b"not a pickle"
+
+
+class TestFilesystemStorageWithDummyPolicy(
+ TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
+):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestFilesystemStorageWithRFC2616Policy(
- TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+ TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
):
- storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
-class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin):
- storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+class TestDbmStorageWithDummyPolicy(
+ TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
+):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestDbmStorageWithRFC2616Policy(
- TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+ TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
):
- storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
@@ -599,3 +653,8 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol
def _get_settings(self, **new_settings) -> dict[str, Any]:
new_settings.setdefault("HTTPCACHE_GZIP", True)
return super()._get_settings(**new_settings)
+
+ def _corrupt_cache_entry(self, storage, spider, request) -> None:
+ # A spider killed while writing a gzip file leaves it truncated.
+ body_path = Path(storage._get_request_path(spider, request), "response_body")
+ body_path.write_bytes(body_path.read_bytes()[:-5])
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index f6f65c3a9..ffc482dce 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -211,6 +211,47 @@ class TestBatchDeliveries(TestFeedExportBase):
header = MyItem.fields.keys()
await self.assertExported(items, header, rows, settings=settings)
+ @coroutine_test
+ async def test_batch_delivered_when_full(self):
+ """Full batches must be finalized and delivered as soon as they are
+ full, instead of when the spider closes."""
+ dir_path = self._random_temp_filename()
+ batch1_path = Path(dir_path, "1.json")
+ mockserver_url = self.mockserver.url("/")
+ batch1_contents: list[bytes | None] = []
+
+ class TestSpider(scrapy.Spider):
+ name = "testspider"
+ start_urls = [mockserver_url]
+
+ def parse(self, response):
+ yield {"foo": "bar1"}
+ yield {"foo": "bar2"}
+ yield scrapy.Request(
+ mockserver_url, callback=self.parse2, dont_filter=True
+ )
+
+ def parse2(self, response):
+ # the first batch was full after the second item, so it must
+ # have been delivered by now
+ batch1_contents.append(
+ batch1_path.read_bytes() if batch1_path.exists() else None
+ )
+ yield {"foo": "bar3"}
+
+ settings = {
+ "FEEDS": {
+ build_url(dir_path / "%(batch_id)d.json"): {"format": "json"},
+ },
+ "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
+ }
+ crawler = get_crawler(TestSpider, settings)
+ await crawler.crawl_async()
+
+ assert batch1_contents, "the second request was not processed"
+ assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl"
+ assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}]
+
def test_wrong_path(self):
"""If path is without %(batch_time)s and %(batch_id) an exception must be raised"""
settings = {
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4bce9a67e..66a9ddb4c 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -33,7 +33,7 @@ from scrapy.pipelines.files import (
GCSFilesStore,
S3FilesStore,
)
-from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
+from scrapy.pipelines.media import _MediaRequestFiltered
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
@@ -43,11 +43,7 @@ from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from .utils.cloud import mock_google_cloud_storage
-from .utils.media_pipelines import mocked_download_func
-
-# required by persist_file() and stat_file(), but as some stores don't use the argument
-# we can pass this singleton to keep type hints correct
-DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider())
+from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func
def get_ftp_content_and_delete(
@@ -94,16 +90,19 @@ class DeferredFSFilesStore(FSFilesStore):
class TestFilesPipeline:
def setup_method(self):
self.tempdir = mkdtemp()
- settings_dict = {"FILES_STORE": self.tempdir}
- crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
- crawler.spider = crawler._create_spider()
- crawler.engine = MagicMock(download_async=mocked_download_func)
- self.pipeline = FilesPipeline.from_crawler(crawler)
- self.pipeline.open_spider()
+ self.pipeline = self._create_pipeline(FilesPipeline)
def teardown_method(self):
rmtree(self.tempdir)
+ def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline:
+ crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir})
+ crawler.spider = crawler._create_spider()
+ crawler.engine = MagicMock(download_async=mocked_download_func)
+ pipeline = pipeline_cls.from_crawler(crawler)
+ pipeline.open_spider()
+ return pipeline
+
@coroutine_test
async def test_close_spider_closes_store(self):
"""close_spider() closes the store when it exposes a close() method."""
@@ -268,6 +267,107 @@ class TestFilesPipeline:
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "cached"
+ @coroutine_test
+ async def test_file_stat_without_last_modified(self) -> None:
+ """A stat result without a last modification time forces a download."""
+ item_url = "http://example.com/file4.pdf"
+ item = _create_item_with_files(item_url)
+ with (
+ mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+ mock.patch.object(
+ FSFilesStore, "stat_file", return_value={"checksum": "abc"}
+ ),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ ):
+ result = await self.pipeline.process_item(item)
+ assert result["files"][0]["checksum"] != "abc"
+ assert result["files"][0]["status"] == "downloaded"
+
+ @coroutine_test
+ async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None:
+ item_url = "http://example.com/empty.pdf"
+ item = _create_item_with_files(item_url)
+ request = Request(
+ item_url, meta={"response": Response(item_url, status=200, body=b"")}
+ )
+ with (
+ caplog.at_level(logging.WARNING),
+ mock.patch.object(
+ FilesPipeline, "get_media_requests", return_value=[request]
+ ),
+ ):
+ result = await self.pipeline.process_item(item)
+ assert result["files"] == []
+ assert "File (empty-content): Empty file from" in caplog.text
+
+ @coroutine_test
+ async def test_file_downloaded_file_exception(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """A FileException from file_downloaded() is logged as a warning and
+ kept as is."""
+
+ class FailingFilesPipeline(FilesPipeline):
+ def file_downloaded(self, response, request, info, *, item=None):
+ raise FileException("boom")
+
+ item_url = "http://example.com/file5.pdf"
+ item = _create_item_with_files(item_url)
+ pipeline = self._create_pipeline(FailingFilesPipeline)
+ with (
+ caplog.at_level(logging.WARNING),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ ):
+ result = await pipeline.process_item(item)
+ assert result["files"] == []
+ records = [
+ r for r in caplog.records if "Error processing file" in r.getMessage()
+ ]
+ assert len(records) == 1
+ assert records[0].levelname == "WARNING"
+ assert "boom" in records[0].getMessage()
+
+ @coroutine_test
+ async def test_file_downloaded_unknown_error(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ """Any other exception from file_downloaded() is logged as an error and
+ reported as a FileException."""
+
+ class FailingFilesPipeline(FilesPipeline):
+ def file_downloaded(self, response, request, info, *, item=None):
+ raise RuntimeError("boom")
+
+ item_url = "http://example.com/file6.pdf"
+ item = _create_item_with_files(item_url)
+ pipeline = self._create_pipeline(FailingFilesPipeline)
+ with (
+ caplog.at_level(logging.WARNING),
+ mock.patch.object(
+ FilesPipeline,
+ "get_media_requests",
+ return_value=[_prepare_request_object(item_url)],
+ ),
+ ):
+ result = await pipeline.process_item(item)
+ assert result["files"] == []
+ records = [
+ r for r in caplog.records if "Error processing file" in r.getMessage()
+ ]
+ assert len(records) == 1
+ assert records[0].levelname == "ERROR"
+ exc_info = records[0].exc_info
+ assert exc_info is not None
+ assert exc_info[0] is RuntimeError
+
@coroutine_test
async def test_async_store(self) -> None:
"""Test that async persist_file() works and is awaited."""
@@ -662,9 +762,24 @@ class TestFilesPipelineCustomSettings:
request = Request("http://example.com/image01.jpg")
assert pipeline.file_path(request) == Path("subdir/image01.jpg")
- def test_files_store_constructor_with_pathlike_object(self, tmp_path):
- fs_store = FSFilesStore(tmp_path)
- assert fs_store.basedir == str(tmp_path)
+
+class TestFSFilesStore:
+ def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None:
+ assert FSFilesStore(tmp_path).basedir == str(tmp_path)
+
+ def test_constructor_with_uri(self, tmp_path: Path) -> None:
+ assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path)
+
+ def test_stat_file(self, tmp_path: Path) -> None:
+ store = FSFilesStore(tmp_path)
+ store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO)
+ stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO)
+ assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc"
+ assert stat["last_modified"] == pytest.approx(time.time(), abs=60)
+
+ def test_stat_missing_file(self, tmp_path: Path) -> None:
+ store = FSFilesStore(tmp_path)
+ assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {}
@pytest.mark.requires_botocore
@@ -712,6 +827,60 @@ class TestS3FilesStore:
# The call to read does not happen with Stubber
assert buffer.method_calls == [mock.call.seek(0)]
+ @inline_callbacks_test
+ def test_persist_without_headers(self, monkeypatch):
+ """Without custom headers only the default ones are sent."""
+ bucket = "mybucket"
+ key = "export.csv"
+ buffer = mock.MagicMock()
+
+ store = S3FilesStore(f"s3://{bucket}/{key}")
+ monkeypatch.setattr(store, "_use_async", lambda: False)
+ from botocore.stub import Stubber # noqa: PLC0415
+
+ with Stubber(store.s3_client) as stub:
+ stub.add_response(
+ "put_object",
+ expected_params={
+ "ACL": S3FilesStore.POLICY,
+ "Body": buffer,
+ "Bucket": bucket,
+ "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
+ "Key": key,
+ "Metadata": {},
+ },
+ service_response={},
+ )
+
+ yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO)
+
+ stub.assert_no_pending_responses()
+
+ def test_missing_botocore(self):
+ with (
+ mock.patch(
+ "scrapy.pipelines.files.is_botocore_available", return_value=False
+ ),
+ pytest.raises(NotConfigured, match="missing botocore library"),
+ ):
+ S3FilesStore("s3://mybucket/key")
+
+ def test_wrong_uri_scheme(self):
+ with pytest.raises(
+ ValueError,
+ match=re.escape(
+ "Incorrect URI scheme in ftp://mybucket/key, expected 's3'"
+ ),
+ ):
+ S3FilesStore("ftp://mybucket/key")
+
+ def test_unsupported_header(self):
+ store = S3FilesStore("s3://mybucket/key")
+ with pytest.raises(
+ TypeError, match='Header "X-Custom" is not supported by botocore'
+ ):
+ store._headers_to_botocore_kwargs({"X-Custom": "value"})
+
@inline_callbacks_test
def test_stat(self, monkeypatch):
"""The blocking botocore client is used when asyncio/aiobotocore support
@@ -1020,6 +1189,28 @@ class TestFTPFileStore:
)
assert data == content
+ @inline_callbacks_test
+ def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch):
+ data = b"active mode"
+ path = "full/filename"
+ monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous")
+ monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest")
+ monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True)
+ with MockFTPServer() as ftp_server:
+ store = FTPFilesStore(ftp_server.url("/"))
+ yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO)
+ stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
+ assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3"
+
+ def test_wrong_uri_scheme(self):
+ with pytest.raises(
+ ValueError,
+ match=re.escape(
+ "Incorrect URI scheme in http://example.com/, expected 'ftp'"
+ ),
+ ):
+ FTPFilesStore("http://example.com/")
+
class ItemWithFiles(Item):
file_urls = Field()
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 1b73dd157..19e61579f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -3,20 +3,26 @@ from __future__ import annotations
import dataclasses
import io
import random
+import sys
from abc import ABC, abstractmethod
+from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
+from types import SimpleNamespace
from typing import Any
import attr
import pytest
from itemadapter import ItemAdapter
+from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.item import Field, Item
-from scrapy.pipelines.files import GCSFilesStore, S3FilesStore
+from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum
from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+from tests.utils.media_pipelines import DUMMY_SPIDER_INFO
try:
from PIL import Image
@@ -40,6 +46,11 @@ class TestImagesPipeline:
def teardown_method(self):
rmtree(self.tempdir)
+ def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setitem(sys.modules, "PIL", None)
+ with pytest.raises(NotConfigured, match="requires installing Pillow"):
+ ImagesPipeline(self.tempdir, crawler=get_crawler())
+
def test_file_path(self):
file_path = self.pipeline.file_path
assert (
@@ -197,6 +208,25 @@ class TestImagesPipeline:
assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
assert new_im.getpixel((0, 0)) == (255, 0, 0)
+ @coroutine_test
+ async def test_image_downloaded(self) -> None:
+ """The image and its thumbnails are stored, and the checksum of the
+ full-size image is returned."""
+ self.pipeline.thumbs = {"small": (20, 20)}
+ _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+ url = "https://dev.mydeco.com/mydeco.gif"
+ response = Response(url=url, body=buf.getvalue())
+
+ checksum = await self.pipeline.image_downloaded(
+ response, Request(url=url), DUMMY_SPIDER_INFO
+ )
+
+ buf.seek(0)
+ assert checksum == _md5sum(buf)
+ name = "3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
+ assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()
+ assert Path(self.tempdir, "thumbs", "small", name).exists()
+
def test_convert_image(self):
SIZE = (100, 100)
# straight forward case: RGB and JPEG
@@ -230,6 +260,24 @@ class TestImagesPipeline:
assert converted.mode == "RGB"
assert converted.getcolors() == [(10000, (205, 230, 255))]
+ def test_convert_image_legacy_resampling_filter(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """Pillow older than 9.1.0 has Image.ANTIALIAS instead of
+ Image.Resampling.LANCZOS."""
+ # Image.LANCZOS is the only spelling that exists in every supported
+ # Pillow version, but Pillow defines it dynamically, hence the ignore.
+ monkeypatch.setattr(
+ self.pipeline,
+ "_Image",
+ SimpleNamespace(ANTIALIAS=Image.LANCZOS), # type: ignore[attr-defined]
+ )
+ im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))
+
+ thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf)
+
+ assert thumbnail.size == (10, 10)
+
@pytest.mark.parametrize(
"bad_type",
[
@@ -581,7 +629,7 @@ class TestImagesPipelineCustomSettings:
GCSFilesStore.POLICY = old_policy
-def _create_image(format_, *a, **kw):
+def _create_image(format_: str, *a: Any, **kw: Any) -> tuple[Image.Image, io.BytesIO]:
buf = io.BytesIO()
Image.new(*a, **kw).save(buf, format_)
buf.seek(0)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 23c19e4be..ba1c18006 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -319,6 +319,42 @@ class TestMediaPipeline(TestBaseMediaPipeline):
assert self.fingerprint(req1) == self.fingerprint(req2)
assert new_item["results"] == [(True, {})]
+ @coroutine_test
+ async def test_failures_are_cached_across_multiple_items(self):
+ self.pipe.LOG_FAILED_RESULTS = False
+ exc = Exception("foo")
+ req1 = Request("http://url1", meta={"response": exc})
+ new_item = await self.pipe.process_item({"requests": req1})
+ assert new_item["results"][0][1].value is exc
+
+ # rsp2 is ignored, the cached failure must be reused because request
+ # fingerprints are the same
+ req2 = Request(
+ req1.url, meta={"response": Response("http://donot.download.me")}
+ )
+ new_item = await self.pipe.process_item({"requests": req2})
+ assert new_item["results"][0][0] is False
+ assert new_item["results"][0][1].value is exc
+ assert self.pipe._mockcalled.count("media_to_download") == 1
+
+ @coroutine_test
+ async def test_cached_failure_calls_errback(self):
+ """The errback of a request is called for a cached failure as well."""
+ self.pipe.LOG_FAILED_RESULTS = False
+ exc = Exception("foo")
+ await self.pipe.process_item(
+ {"requests": Request("http://url1", meta={"response": exc})}
+ )
+
+ def errback(failure):
+ self.pipe._mockcalled.append("request_errback")
+ return {"recovered": failure.value}
+
+ req = Request("http://url1", errback=errback)
+ new_item = await self.pipe.process_item({"requests": req})
+ assert new_item["results"] == [(True, {"recovered": exc})]
+ assert self.pipe._mockcalled.count("request_errback") == 1
+
@coroutine_test
async def test_results_are_cached_for_requests_of_single_item(self):
rsp1 = Response("http://url1")
@@ -472,6 +508,30 @@ class TestBuildFromCrawler:
assert pipe._from_crawler_called
+class MediaFailedNonePipeline(MockedMediaPipeline):
+ def media_failed(self, failure, request, info):
+ self._mockcalled.append("media_failed")
+
+
+class TestMediaFailedNone(TestBaseMediaPipeline):
+ """Test what happens when media_failed() neither raises an exception nor
+ returns a failure."""
+
+ pipeline_class = MediaFailedNonePipeline
+
+ @coroutine_test
+ async def test_result_none(self):
+ req = Request("http://url1", meta={"response": Exception("foo")})
+ new_item = await self.pipe.process_item({"requests": req})
+ assert new_item["results"] == [(True, None)]
+ assert self.pipe._mockcalled == [
+ "get_media_requests",
+ "media_to_download",
+ "media_failed",
+ "item_completed",
+ ]
+
+
class MediaFailedFailurePipeline(MockedMediaPipeline):
def media_failed(self, failure, request, info):
self._mockcalled.append("media_failed")
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 03d17199f..38cb8da18 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,11 +1,26 @@
from __future__ import annotations
+from typing import TYPE_CHECKING
+
import pytest
-from scrapy.http import Response, TextResponse, XmlResponse
+from scrapy.http import Request, Response, TextResponse, XmlResponse
from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
from tests import get_testdata
+from tests.spiders import RawResponseSpider
from tests.utils.bases.spider import TestSpiderBase
+from tests.utils.crawl import crawl_items
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+ from tests.mockserver.http import MockServer
+
+
+class RawFeedSpider(RawResponseSpider):
+ content_type = "text/xml"
+
+ async def start(self):
+ yield Request(self.raw_url)
class TestSpider(TestSpiderBase):
@@ -60,6 +75,89 @@ class TestXMLFeedSpider(TestSpiderBase):
},
], iterator
+ @coroutine_test
+ async def test_parse_node_uses_parse_item(self, mockserver: MockServer):
+ # parse_node falls back to parse_item for backward compatibility.
+ class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
+ itertag = "item"
+
+ def raw_body(self):
+ return "- 1
"
+
+ def parse_item(self, response, selector):
+ return {"id": selector.xpath("id/text()").get()}
+
+ items, _ = await crawl_items(_Spider, mockserver)
+ assert items == [{"id": "1"}]
+
+ @coroutine_test
+ async def test_parse_node_not_defined(self, mockserver: MockServer):
+ class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
+ itertag = "item"
+
+ def raw_body(self):
+ return "- 1
"
+
+ items, crawler = await crawl_items(_Spider, mockserver)
+ assert items == []
+ assert crawler.stats
+ assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
+
+ @coroutine_test
+ async def test_html_iterator(self, mockserver: MockServer):
+ class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
+ iterator = "html"
+ itertag = "item"
+ content_type = "text/html"
+
+ def raw_body(self):
+ return (
+ "- 1
"
+ "- 2
"
+ )
+
+ def parse_node(self, response, selector):
+ return {"id": selector.xpath("id/text()").get()}
+
+ items, _ = await crawl_items(_Spider, mockserver)
+ assert items == [{"id": "1"}, {"id": "2"}]
+
+ @coroutine_test
+ async def test_unsupported_iterator(self, mockserver: MockServer):
+ class _Spider(RawFeedSpider, self.spider_class): # type: ignore[name-defined,misc]
+ iterator = "unsupported"
+
+ def raw_body(self):
+ return "