RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: simplify some code, get rid of nested fns where it's makes sence + pylint (#7401) 

* chore: simplify some code, get rid of nested fns where it's makes sence

reasoning: https://wemake-python-styleguide.readthedocs.io/en/latest/pages/usage/violations/best_practices.html#wemake_python_styleguide.violations.best_practices.NestedFunctionViolation

* chore: address some pylint rules

* remove `pylint: disable=deprecated-class` from `reactor.py`

* fix typo

* fix another typo :)

* revert backward incompatible arg renames

---------

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
This commit is contained in:
Albert Eduardovich N. 2026-05-04 21:32:46 +03:00 committed by GitHub
parent 8d69a7c865
commit 9776a72a6a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 350 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
extras/qps-bench-server.py
View File

@ -18,7 +18,7 @@ class Root(Resource):
self.tail.clear()
self.start = self.lastmark = self.lasttime = time()
def getChild(self, request, name):
def getChild(self, path, request):
return self
def render(self, request):

13
pyproject.toml
View File

@ -185,6 +185,7 @@ jobs = 1 # >1 hides results
extension-pkg-allow-list=[
"lxml",
]
load-plugins = ["pylint_per_file_ignores"]
[tool.pylint."MESSAGES CONTROL"]
enable = [
@ -244,15 +245,13 @@ disable = [
"unused-import",
# Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
"abstract-method",
"arguments-differ",
"arguments-renamed",
"dangerous-default-value",
"keyword-arg-before-vararg",
"pointless-statement",
"raise-missing-from",
"unnecessary-dunder-call",
"used-before-assignment",
]
# requires `pylint_per_file_ignores` plugin
per-file-ignores = [
# Extended list of ones that we may want to address, only for tests
"./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment",
]
[tool.pytest.ini_options]

2
scrapy/contracts/__init__.py
View File

@ -125,7 +125,7 @@ class ContractsManager:
def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]:
requests: list[Request | None] = []
for method in self.tested_methods_from_spidercls(type(spider)):
bound_method = spider.__getattribute__(method)
bound_method = getattr(spider, method)
try:
requests.append(self.from_method(bound_method, results))
except Exception:

20
scrapy/core/downloader/handlers/http11.py
View File

@ -6,6 +6,7 @@ import ipaddress
import logging
import re
from contextlib import suppress
from functools import partial
from io import BytesIO
from time import monotonic
from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
@ -547,11 +548,7 @@ class ScrapyAgent:
get_warnsize_msg(expected_size, warnsize, request, expected=True)
)
def _cancel(_: Any) -> None:
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
d: Deferred[_ResultT] = Deferred(_cancel)
d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse))
txresponse.deliverBody(
_ResponseReader(
finished=d,
@ -570,6 +567,11 @@ class ScrapyAgent:
return d
@staticmethod
def _cancel(_: Any, txresponse: TxResponse) -> None:
# Abort connection immediately.
txresponse._transport._producer.abortConnection()
def _cb_bodydone(self, result: _ResultT, url: str) -> Response:
headers = self._headers_from_twisted_response(result["txresponse"])
try:
@ -667,17 +669,17 @@ class _ResponseReader(Protocol):
assert hostname is not None
_log_ssl_conn_debug_info(hostname, connection)
def dataReceived(self, bodyBytes: bytes) -> None:
def dataReceived(self, data: bytes) -> None:
# This maybe called several times after cancel was called with buffered data.
if self._finished.called:
return
assert self.transport
self._bodybuf.write(bodyBytes)
self._bytes_received += len(bodyBytes)
self._bodybuf.write(data)
self._bytes_received += len(data)
if stop_download := check_stop_download(
signals.bytes_received, self._crawler, self._request, data=bodyBytes
signals.bytes_received, self._crawler, self._request, data=data
):
self.transport.stopProducing()
self.transport.loseConnection()

151
scrapy/core/downloader/middleware.py
View File

@ -75,87 +75,82 @@ class DownloaderMiddlewareManager(MiddlewareManager):
download_func: Callable[[Request], Coroutine[Any, Any, Response]],
request: Request,
) -> Response | Request:
async def process_request(request: Request) -> Response | Request:
for method in self.methods["process_request"]:
method = cast("Callable", method)
if method in self._mw_methods_requiring_spider:
response = await ensure_awaitable(
method(request=request, spider=self._spider),
_warn=global_object_name(method),
)
else:
response = await ensure_awaitable(
method(request=request), _warn=global_object_name(method)
)
if response is not None and not isinstance(
response, (Response, Request)
):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {response.__class__.__name__}"
)
if response:
return response
return await download_func(request)
async def process_response(response: Response | Request) -> Response | Request:
if response is None:
raise TypeError("Received None in process_response")
if isinstance(response, Request):
return response
for method in self.methods["process_response"]:
method = cast("Callable", method)
if method in self._mw_methods_requiring_spider:
response = await ensure_awaitable(
method(request=request, response=response, spider=self._spider),
_warn=global_object_name(method),
)
else:
response = await ensure_awaitable(
method(request=request, response=response),
_warn=global_object_name(method),
)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return Response or Request, "
f"got {type(response)}"
)
if isinstance(response, Request):
return response
return response
async def process_exception(exception: Exception) -> Response | Request:
for method in self.methods["process_exception"]:
method = cast("Callable", method)
if method in self._mw_methods_requiring_spider:
response = await ensure_awaitable(
method(
request=request, exception=exception, spider=self._spider
),
_warn=global_object_name(method),
)
else:
response = await ensure_awaitable(
method(request=request, exception=exception),
_warn=global_object_name(method),
)
if response is not None and not isinstance(
response, (Response, Request)
):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {type(response)}"
)
if response:
return response
raise exception
try:
result: Response | Request = await process_request(request)
result: Response | Request = await self._process_request(
request, download_func
)
except Exception as ex:
await _defer_sleep_async()
# either returns a request or response (which we pass to process_response())
# or reraises the exception
result = await process_exception(ex)
return await process_response(result)
result = await self._process_exception(ex, request)
return await self._process_response(result, request)
def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
if method in self._mw_methods_requiring_spider:
kwargs["spider"] = self._spider
return method(**kwargs)
async def _process_request(
self,
request: Request,
download_func: Callable[[Request], Coroutine[Any, Any, Response]],
) -> Response | Request:
for method in self.methods["process_request"]:
method = cast("Callable", method)
response = await ensure_awaitable(
self._handle_mw_method(method, request=request),
_warn=global_object_name(method),
)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {response.__class__.__name__}"
)
if response:
return response
return await download_func(request)
async def _process_response(
self, response: Response | Request, request: Request
) -> Response | Request:
if response is None:
raise TypeError("Received None in process_response")
if isinstance(response, Request):
return response
for method in self.methods["process_response"]:
method = cast("Callable", method)
response = await ensure_awaitable(
self._handle_mw_method(method, request=request, response=response),
_warn=global_object_name(method),
)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return Response or Request, "
f"got {type(response)}"
)
if isinstance(response, Request):
return response
return response
async def _process_exception(
self, exception: Exception, request: Request | Response
) -> Response | Request:
for method in self.methods["process_exception"]:
method = cast("Callable", method)
response = await ensure_awaitable(
self._handle_mw_method(method, request=request, exception=exception),
_warn=global_object_name(method),
)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput(
f"Middleware {method.__qualname__} must return None, Response or "
f"Request, got {type(response)}"
)
if response:
return response
raise exception

40
scrapy/core/engine.py
View File

@ -11,6 +11,7 @@ import asyncio
import contextlib
import logging
import warnings
from functools import partial
from time import time
from traceback import format_exc
from typing import TYPE_CHECKING, Any
@ -273,7 +274,7 @@ class ExecutionEngine:
"""
assert self._start is not None
try:
item_or_request = await self._start.__anext__()
item_or_request = await anext(self._start)
except StopAsyncIteration:
self._start = None
except Exception as exception:
@ -352,6 +353,10 @@ class ExecutionEngine:
or self.scraper.slot.needs_backout()
)
def _remove_request(self, _: Any, request: Request) -> None:
assert self._slot
self._slot.remove_request(request)
def _start_scheduled_request(self) -> bool:
assert self._slot is not None # typing
assert self.spider is not None # typing
@ -371,11 +376,7 @@ class ExecutionEngine:
)
)
def _remove_request(_: Any) -> None:
assert self._slot
self._slot.remove_request(request)
d2: Deferred[None] = d.addBoth(_remove_request)
d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request))
d2.addErrback(
lambda f: logger.info(
"Error while removing request from slot",
@ -612,30 +613,33 @@ class ExecutionEngine:
"Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
)
def log_failure(msg: str) -> None:
logger.error(msg, exc_info=True, extra={"spider": spider}) # noqa: LOG014
try:
await self._slot.close()
except Exception:
log_failure("Slot close failure")
logger.error("Slot close failure", exc_info=True, extra={"spider": spider})
try:
self.downloader.close()
except Exception:
log_failure("Downloader close failure")
logger.error(
"Downloader close failure", exc_info=True, extra={"spider": spider}
)
try:
await self.scraper.close_spider_async()
except Exception:
log_failure("Scraper close failure")
logger.error(
"Scraper close failure", exc_info=True, extra={"spider": spider}
)
if hasattr(self._slot.scheduler, "close"):
try:
if (d := self._slot.scheduler.close(reason)) is not None:
await maybe_deferred_to_future(d)
except Exception:
log_failure("Scheduler close failure")
logger.error(
"Scheduler close failure", exc_info=True, extra={"spider": spider}
)
try:
await self.signals.send_catch_log_async(
@ -644,7 +648,11 @@ class ExecutionEngine:
reason=reason,
)
except Exception:
log_failure("Error while sending spider_close signal")
logger.error(
"Error while sending spider_close signal",
exc_info=True,
extra={"spider": spider},
)
assert self.crawler.stats
try:
@ -661,7 +669,7 @@ class ExecutionEngine:
else:
self.crawler.stats.close_spider(reason=reason)
except Exception:
log_failure("Stats close failure")
logger.error("Stats close failure")
logger.info(
"Spider closed (%(reason)s)",
@ -675,4 +683,4 @@ class ExecutionEngine:
try:
await ensure_awaitable(self._spider_closed_callback(spider))
except Exception:
log_failure("Error running spider_closed_callback")
logger.error("Error running spider_closed_callback")

18
scrapy/core/http2/stream.py
View File

@ -155,16 +155,16 @@ class Stream:
"status": None,
}
def _cancel(_: Any) -> None:
# Close this stream as gracefully as possible
# If the associated request is initiated we reset this stream
# else we directly call close() method
if self.metadata["request_sent"]:
self.reset_stream(StreamCloseReason.CANCELLED)
else:
self.close(StreamCloseReason.CANCELLED)
self._deferred_response: Deferred[Response] = Deferred(self._cancel)
self._deferred_response: Deferred[Response] = Deferred(_cancel)
def _cancel(self, _: Any) -> None:
# Close this stream as gracefully as possible
# If the associated request is initiated we reset this stream
# else we directly call close() method
if self.metadata["request_sent"]:
self.reset_stream(StreamCloseReason.CANCELLED)
else:
self.close(StreamCloseReason.CANCELLED)
def __repr__(self) -> str:
return f"Stream(id={self.stream_id!r})"

98
scrapy/core/spidermw.py
View File

@ -112,40 +112,59 @@ class SpiderMiddlewareManager(MiddlewareManager):
exception_processor_index: int,
recover_to: MutableChain[_T] | MutableAsyncChain[_T],
) -> Iterable[_T] | AsyncIterator[_T]:
def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
try:
yield from iterable
except Exception as ex:
exception_result = cast(
"Failure | MutableChain[_T]",
self._process_spider_exception(
response, ex, exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableChain)
recover_to.extend(exception_result)
async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]:
try:
async for r in iterable:
yield r
except Exception as ex:
exception_result = cast(
"Failure | MutableAsyncChain[_T]",
self._process_spider_exception(
response, ex, exception_processor_index
),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableAsyncChain)
recover_to.extend(exception_result)
if isinstance(iterable, AsyncIterator):
return process_async(iterable)
return process_sync(iterable)
return self._process_async(
response,
iterable,
exception_processor_index,
cast("MutableAsyncChain[_T]", recover_to),
)
return self._process_sync(
response,
iterable,
exception_processor_index,
cast("MutableChain[_T]", recover_to),
)
def _process_sync(
self,
response: Response,
iterable: Iterable[_T],
exception_processor_index: int,
recover_to: MutableChain[_T],
) -> Iterable[_T]:
try:
yield from iterable
except Exception as ex:
exception_result = cast(
"Failure | MutableChain[_T]",
self._process_spider_exception(response, ex, exception_processor_index),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableChain)
recover_to.extend(exception_result)
async def _process_async(
self,
response: Response,
iterable: AsyncIterator[_T],
exception_processor_index: int,
recover_to: MutableAsyncChain[_T],
) -> AsyncIterator[_T]:
try:
async for r in iterable:
yield r
except Exception as ex:
exception_result = cast(
"Failure | MutableAsyncChain[_T]",
self._process_spider_exception(response, ex, exception_processor_index),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableAsyncChain)
recover_to.extend(exception_result)
def _process_spider_exception(
self,
@ -350,25 +369,14 @@ class SpiderMiddlewareManager(MiddlewareManager):
"scrape_response_async() called on a SpiderMiddlewareManager"
" instance created without a crawler."
)
async def process_callback_output(
result: Iterable[_T] | AsyncIterator[_T],
) -> MutableChain[_T] | MutableAsyncChain[_T]:
return await self._process_callback_output(response, result)
def process_spider_exception(
exception: Exception,
) -> MutableChain[_T] | MutableAsyncChain[_T]:
return self._process_spider_exception(response, exception)
try:
it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input(
scrape_func, response, request
)
return await process_callback_output(it)
return await self._process_callback_output(response, it)
except Exception as ex:
await _defer_sleep_async()
return process_spider_exception(ex)
return self._process_spider_exception(response, ex)
async def process_start(
self, spider: Spider | None = None

46
scrapy/crawler.py
View File

@ -7,6 +7,7 @@ import pprint
import signal
import warnings
from abc import ABC, abstractmethod
from functools import partial
from typing import TYPE_CHECKING, Any, TypeVar
from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
@ -568,28 +569,30 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
crawler = self.create_crawler(crawler_or_spidercls)
return self._crawl(crawler, *args, **kwargs)
async def _crawl_and_track(
self, crawler: Crawler, *args: Any, **kwargs: Any
) -> None:
try:
await crawler.crawl_async(*args, **kwargs)
except Exception:
self.bootstrap_failed = True
raise # re-raise so asyncio still logs it to stderr naturally
def _done(self, task: asyncio.Task[None], crawler: Crawler) -> None:
self._active.discard(task)
self.crawlers.discard(crawler)
self.bootstrap_failed |= not getattr(crawler, "spider", None)
def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:
# At this point the asyncio loop has been installed either by the user
# or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
loop = asyncio.get_event_loop()
self.crawlers.add(crawler)
async def _crawl_and_track() -> None:
try:
await crawler.crawl_async(*args, **kwargs)
except Exception:
self.bootstrap_failed = True
raise # re-raise so asyncio still logs it to stderr naturally
task = loop.create_task(_crawl_and_track())
task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs))
self._active.add(task)
task.add_done_callback(partial(self._done, crawler=crawler))
def _done(_: asyncio.Task[None]) -> None:
self.crawlers.discard(crawler)
self._active.discard(task)
self.bootstrap_failed |= not getattr(crawler, "spider", None)
task.add_done_callback(_done)
return task
async def stop(self) -> None:
@ -1007,14 +1010,15 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
if (loop := self._reactorless_loop) is None:
return
def _create_shutdown_task() -> None:
coro = self._shutdown_graceful_reactorless()
try:
loop.create_task(coro)
except RuntimeError:
coro.close()
loop.call_soon_threadsafe(self._create_shutdown_task)
loop.call_soon_threadsafe(_create_shutdown_task)
def _create_shutdown_task(self) -> None:
assert self._reactorless_loop
coro = self._shutdown_graceful_reactorless()
try:
self._reactorless_loop.create_task(coro)
except RuntimeError:
coro.close()
async def _shutdown_graceful_reactorless(self) -> None:
await self.stop()

16
scrapy/extensions/feedexport.py
View File

@ -534,13 +534,15 @@ class FeedExporter:
# Send FEED_EXPORTER_CLOSED signal
await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
@staticmethod
def _get_file(slot_: FeedSlot) -> IO[bytes]:
assert slot_.file
if isinstance(slot_.file, PostProcessingManager):
slot_.file.close()
return slot_.file.file
return slot_.file
async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None:
def get_file(slot_: FeedSlot) -> IO[bytes]:
assert slot_.file
if isinstance(slot_.file, PostProcessingManager):
slot_.file.close()
return slot_.file.file
return slot_.file
if slot.itemcount:
# Normal case
@ -557,7 +559,7 @@ class FeedExporter:
slot_type = type(slot.storage).__name__
assert self.crawler.stats
try:
await ensure_awaitable(slot.storage.store(get_file(slot)))
await ensure_awaitable(slot.storage.store(self._get_file(slot)))
except Exception:
logger.error(
"Error storing %s",

69
scrapy/pipelines/__init__.py
View File

@ -37,16 +37,16 @@ class ItemPipelineManager(MiddlewareManager):
settings.get_component_priority_dict_with_base("ITEM_PIPELINES")
)
def _add_middleware(self, pipe: Any) -> None:
if hasattr(pipe, "open_spider"):
self.methods["open_spider"].append(pipe.open_spider)
self._check_mw_method_spider_arg(pipe.open_spider)
if hasattr(pipe, "close_spider"):
self.methods["close_spider"].appendleft(pipe.close_spider)
self._check_mw_method_spider_arg(pipe.close_spider)
if hasattr(pipe, "process_item"):
self.methods["process_item"].append(pipe.process_item)
self._check_mw_method_spider_arg(pipe.process_item)
def _add_middleware(self, mw: Any) -> None:
if hasattr(mw, "open_spider"):
self.methods["open_spider"].append(mw.open_spider)
self._check_mw_method_spider_arg(mw.open_spider)
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
self._check_mw_method_spider_arg(mw.close_spider)
if hasattr(mw, "process_item"):
self.methods["process_item"].append(mw.process_item)
self._check_mw_method_spider_arg(mw.process_item)
def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:
warnings.warn(
@ -62,32 +62,44 @@ class ItemPipelineManager(MiddlewareManager):
"process_item", item, add_spider=True, warn_deferred=True
)
def _get_dfd(
self,
method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
) -> Deferred[None]:
if method in self._mw_methods_requiring_spider:
return _maybeDeferred_coro(method, True, self._spider)
return _maybeDeferred_coro(method, True)
@staticmethod
def _eb(failure: Failure) -> Failure:
assert isinstance(failure.value, FirstError)
return failure.value.subFailure
def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]:
methods = cast(
"Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
self.methods[methodname],
)
def get_dfd(
method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
) -> Deferred[None]:
if method in self._mw_methods_requiring_spider:
return _maybeDeferred_coro(method, True, self._spider)
return _maybeDeferred_coro(method, True)
dfds = [get_dfd(m) for m in methods]
dfds = [self._get_dfd(m) for m in methods]
d: Deferred[list[tuple[bool, None]]] = DeferredList(
dfds, fireOnOneErrback=True, consumeErrors=True
)
d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
def eb(failure: Failure) -> Failure:
assert isinstance(failure.value, FirstError)
return failure.value.subFailure
d2.addErrback(eb)
d2.addErrback(self._eb)
return d2
def get_awaitable(
self,
method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
) -> Awaitable[None]:
if method in self._mw_methods_requiring_spider:
result = method(self._spider)
else:
result = method()
return ensure_awaitable(result, _warn=global_object_name(method))
async def _process_parallel_asyncio(self, methodname: str) -> list[None]:
methods = cast(
"Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
@ -96,16 +108,7 @@ class ItemPipelineManager(MiddlewareManager):
if not methods:
return []
def get_awaitable(
method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
) -> Awaitable[None]:
if method in self._mw_methods_requiring_spider:
result = method(self._spider)
else:
result = method()
return ensure_awaitable(result, _warn=global_object_name(method))
awaitables = [get_awaitable(m) for m in methods]
awaitables = [self.get_awaitable(m) for m in methods]
await asyncio.gather(*awaitables)
return [None for _ in methods]

134
scrapy/pipelines/files.py
View File

@ -186,16 +186,18 @@ class S3FilesStore:
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
self.bucket, self.prefix = uri[5:].split("/", 1)
@staticmethod
def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
checksum = boto_key["ETag"].strip('"')
last_modified = boto_key["LastModified"]
modified_stamp = time.mktime(last_modified.timetuple())
return {"checksum": checksum, "last_modified": modified_stamp}
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Deferred[StatInfo]:
def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
checksum = boto_key["ETag"].strip('"')
last_modified = boto_key["LastModified"]
modified_stamp = time.mktime(last_modified.timetuple())
return {"checksum": checksum, "last_modified": modified_stamp}
return self._get_boto_key(path).addCallback(_onsuccess)
return self._get_boto_key(path).addCallback(self._onsuccess)
def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]:
key_name = f"{self.prefix}{path}"
@ -308,20 +310,22 @@ class GCSFilesStore:
{"bucket": bucket},
)
@staticmethod
def _onsuccess(blob: Any) -> StatInfo:
if blob:
checksum = base64.b64decode(blob.md5_hash).hex()
last_modified = time.mktime(blob.updated.timetuple())
return {"checksum": checksum, "last_modified": last_modified}
return {}
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Deferred[StatInfo]:
def _onsuccess(blob: Any) -> StatInfo:
if blob:
checksum = base64.b64decode(blob.md5_hash).hex()
last_modified = time.mktime(blob.updated.timetuple())
return {"checksum": checksum, "last_modified": last_modified}
return {}
blob_path = self._get_blob_path(path)
return deferred_from_coro(
run_in_thread(self.bucket.get_blob, blob_path)
).addCallback(_onsuccess)
).addCallback(self._onsuccess)
def _get_content_type(self, headers: dict[str, str] | None) -> str:
if headers and "Content-Type" in headers:
@ -395,26 +399,26 @@ class FTPFilesStore:
)
)
def _stat_file(self, path: str) -> StatInfo:
try:
with FTP() as ftp:
ftp.connect(self.host, self.port)
ftp.login(self.username, self.password)
if self.USE_ACTIVE_MODE:
ftp.set_pasv(False)
file_path = f"{self.basedir}/{path}"
last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
m = hashlib.md5() # noqa: S324
ftp.retrbinary(f"RETR {file_path}", m.update)
return {"last_modified": last_modified, "checksum": m.hexdigest()}
# The file doesn't exist
except Exception:
return {}
def stat_file(
self, path: str, info: MediaPipeline.SpiderInfo
) -> Deferred[StatInfo]:
def _stat_file(path: str) -> StatInfo:
try:
with FTP() as ftp:
ftp.connect(self.host, self.port)
ftp.login(self.username, self.password)
if self.USE_ACTIVE_MODE:
ftp.set_pasv(False)
file_path = f"{self.basedir}/{path}"
last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
m = hashlib.md5() # noqa: S324
ftp.retrbinary(f"RETR {file_path}", m.update)
return {"last_modified": last_modified, "checksum": m.hexdigest()}
# The file doesn't exist
except Exception:
return {}
return deferred_from_coro(run_in_thread(_stat_file, path))
return deferred_from_coro(run_in_thread(self._stat_file, path))
class FilesPipeline(MediaPipeline):
@ -534,43 +538,51 @@ class FilesPipeline(MediaPipeline):
store_cls = self.STORE_SCHEMES[scheme]
return store_cls(uri)
def _onsuccess(
self,
result: StatInfo,
request: Request,
info: MediaPipeline.SpiderInfo,
path: str,
) -> FileInfo | None:
if not result:
return None # returning None force download
last_modified = result.get("last_modified", None)
if not last_modified:
return None # returning None force download
age_seconds = time.time() - last_modified
age_days = age_seconds / 60 / 60 / 24
if age_days > self.expires:
return None # returning None force download
referer = referer_str(request)
logger.debug(
"File (uptodate): Downloaded %(medianame)s from %(request)s "
"referred in <%(referer)s>",
{"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
extra={"spider": info.spider},
)
self.inc_stats("uptodate")
checksum = result.get("checksum", None)
return {
"url": request.url,
"path": path,
"checksum": checksum,
"status": "uptodate",
}
def media_to_download(
self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
) -> Deferred[FileInfo | None] | None:
def _onsuccess(result: StatInfo) -> FileInfo | None:
if not result:
return None # returning None force download
last_modified = result.get("last_modified", None)
if not last_modified:
return None # returning None force download
age_seconds = time.time() - last_modified
age_days = age_seconds / 60 / 60 / 24
if age_days > self.expires:
return None # returning None force download
referer = referer_str(request)
logger.debug(
"File (uptodate): Downloaded %(medianame)s from %(request)s "
"referred in <%(referer)s>",
{"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
extra={"spider": info.spider},
)
self.inc_stats("uptodate")
checksum = result.get("checksum", None)
return {
"url": request.url,
"path": path,
"checksum": checksum,
"status": "uptodate",
}
path = self.file_path(request, info=info, item=item)
# maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload]
dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess)
dfd2: Deferred[FileInfo | None] = dfd.addCallback(
functools.partial(self._onsuccess, request=request, info=info, path=path)
)
dfd2.addErrback(lambda _: None)
dfd2.addErrback(
lambda f: logger.error(

8
scrapy/settings/__init__.py
View File

@ -143,7 +143,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
def get(self, name: _SettingsKey, default: Any = None) -> Any:
def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed
"""
Get a setting value without affecting its original type.
@ -510,7 +510,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
component_priority_dict[cls] = priority
self.set(name, component_priority_dict, self.getpriority(name) or 0)
def setdefault(
def setdefault( # pylint: disable=arguments-renamed
self,
name: _SettingsKey,
default: Any = None,
@ -691,14 +691,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
else:
p.text(pformat(self.copy_to_dict()))
def pop(self, name: _SettingsKey, default: Any = __default) -> Any:
def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed
try:
value = self.attributes[name].value
except KeyError:
if default is self.__default:
raise
return default
self.__delitem__(name)
del self[name]
return value

2
scrapy/utils/benchserver.py
View File

@ -9,7 +9,7 @@ from twisted.web.server import Request, Site
class Root(Resource):
isLeaf = True
def getChild(self, name: str, request: Request) -> Resource:
def getChild(self, path: str, request: Request) -> Resource:
return self
def render(self, request: Request) -> bytes:

6
scrapy/utils/defer.py
View File

@ -262,7 +262,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
def _call_anext(self) -> None:
# This starts waiting for the next result from aiterator.
# If aiterator is exhausted, _errback will be called.
self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
self.anext_deferred = deferred_from_coro(anext(self.aiterator))
self.anext_deferred.addCallbacks(self._callback, self._errback)
def __next__(self) -> Deferred[Any]:
@ -370,10 +370,10 @@ async def aiter_errback(
"""Wrap an async iterable calling an errback if an error is caught while
iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`.
"""
it = aiterable.__aiter__()
it = aiter(aiterable)
while True:
try:
yield await it.__anext__()
yield await anext(it)
except StopAsyncIteration:
break
except Exception:

13
scrapy/utils/misc.py
View File

@ -255,6 +255,11 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
def _returns_none(return_node: ast.Return) -> bool:
value = return_node.value
return value is None or (isinstance(value, ast.Constant) and value.value is None)
def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noqa: A002
"""
Returns True if a callable is a generator function which includes a
@ -263,12 +268,6 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noq
if callable in _generator_callbacks_cache:
return bool(_generator_callbacks_cache[callable])
def returns_none(return_node: ast.Return) -> bool:
value = return_node.value
return value is None or (
isinstance(value, ast.Constant) and value.value is None
)
if inspect.isgeneratorfunction(callable):
func = callable
while isinstance(func, partial):
@ -284,7 +283,7 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noq
tree = ast.parse(code)
for node in walk_callable(tree):
if isinstance(node, ast.Return) and not returns_none(node):
if isinstance(node, ast.Return) and not _returns_none(node):
_generator_callbacks_cache[callable] = True
return bool(_generator_callbacks_cache[callable])

21
scrapy/utils/python.py
View File

@ -99,6 +99,16 @@ def to_bytes(
return text.encode(encoding, errors)
def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:
offset = len(text)
while True:
offset -= chunk_size * 1024
if offset <= 0:
break
yield (text[offset:], offset)
yield (text, 0)
def re_rsearch(
pattern: str | Pattern[str], text: str, chunk_size: int = 1024
) -> tuple[int, int] | None:
@ -115,19 +125,10 @@ def re_rsearch(
the start position of the match, and the ending (regarding the entire text).
"""
def _chunk_iter() -> Iterable[tuple[str, int]]:
offset = len(text)
while True:
offset -= chunk_size * 1024
if offset <= 0:
break
yield (text[offset:], offset)
yield (text, 0)
if isinstance(pattern, str):
pattern = re.compile(pattern)
for chunk, offset in _chunk_iter():
for chunk, offset in _chunk_iter(text, chunk_size):
matches = list(pattern.finditer(chunk))
if matches:
start, end = matches[-1].span()

2
tests/mockserver/http.py
View File

@ -85,7 +85,7 @@ class Root(resource.Resource):
self.putChild(b"response-headers", ResponseHeadersResource())
self.putChild(b"set-cookie", SetCookie())
def getChild(self, name, request):
def getChild(self, path, request):
return self
def render(self, request):

2
tests/mockserver/simple_https.py
View File

@ -13,7 +13,7 @@ class Root(resource.Resource):
resource.Resource.__init__(self)
self.putChild(b"file", Data(b"0123456789", "text/plain"))
def getChild(self, name, request):
def getChild(self, path, request):
return self

3
tox.ini
View File

@ -89,8 +89,9 @@ basepython = python3
deps =
{[testenv:extra-deps]deps}
pylint==4.0.2
pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278
commands =
pylint conftest.py docs extras scrapy tests
pylint {posargs:conftest.py docs extras scrapy tests}
[testenv:twinecheck]
basepython = python3