RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into throttling

This commit is contained in:
Adrian Chaves 2026-06-30 16:30:24 +02:00
parent b3ea60354f a6d6a48aa6
commit c30fcc2c16
49 changed files with 326 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/addons/__init__.py
View File

@ -64,7 +64,7 @@ class AddonManager:
:param settings: The :class:`~scrapy.settings.BaseSettings` object from \
which to read the early add-on configuration
:type settings: :class:`~scrapy.settings.Settings`
:type settings: :class:`~scrapy.settings.BaseSettings`
"""
for clspath in build_component_list(settings["ADDONS"]):
addoncls = load_object(clspath)

2
scrapy/commands/__init__.py
View File

@ -73,7 +73,7 @@ class ScrapyCommand(ABC):
def long_desc(self) -> str:
"""A long description of the command. Return short description when not
available. It cannot contain newlines since contents will be formatted
by optparser which removes newlines and wraps text.
by argparse which removes newlines and wraps text.
"""
return self.short_desc()

23
scrapy/commands/edit.py
View File

@ -1,12 +1,29 @@
import argparse
from __future__ import annotations
import os
import shlex
import subprocess
import sys
from typing import Any, ClassVar
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
from scrapy.spiderloader import get_spider_loader
if TYPE_CHECKING:
import argparse
def _edit_file(editor: str, file_path: str | os.PathLike[str]) -> int:
"""Open ``file_path`` with ``editor`` and return the editor exit code.
``editor`` may include arguments (e.g. ``"code -w"``); it is split with
:func:`shlex.split` and the file is passed as a separate argument, so no
shell is involved.
"""
return subprocess.call([*shlex.split(editor), os.fspath(file_path)]) # noqa: S603
class Command(ScrapyCommand):
requires_project = True
@ -45,4 +62,4 @@ class Command(ScrapyCommand):
sfile = sys.modules[spidercls.__module__].__file__
assert sfile
sfile = sfile.replace(".pyc", ".py")
self.exitcode = os.system(f'{editor} "{sfile}"') # noqa: S605
self.exitcode = _edit_file(editor, Path(sfile))

12
scrapy/commands/genspider.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations
import os
import shutil
import string
from importlib import import_module
@ -10,12 +9,14 @@ from urllib.parse import urlparse
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.commands.edit import _edit_file
from scrapy.exceptions import UsageError
from scrapy.spiderloader import get_spider_loader
from scrapy.utils.template import render_templatefile, string_camelcase
if TYPE_CHECKING:
import argparse
import os
def sanitize_module_name(module_name: str) -> str:
@ -118,9 +119,11 @@ class Command(ScrapyCommand):
template_file = self._find_template(opts.template)
if template_file:
self._genspider(module, name, url, opts.template, template_file)
spider_file = self._genspider(
module, name, url, opts.template, template_file
)
if opts.edit:
self.exitcode = os.system(f'scrapy edit "{name}"') # noqa: S605
self.exitcode = _edit_file(self.settings["EDITOR"], spider_file)
def _generate_template_variables(
self,
@ -148,7 +151,7 @@ class Command(ScrapyCommand):
url: str,
template_name: str,
template_file: str | os.PathLike[str],
) -> None:
) -> Path:
"""Generate the spider module, based on the given template"""
assert self.settings is not None
tvars = self._generate_template_variables(module, name, url, template_name)
@ -168,6 +171,7 @@ class Command(ScrapyCommand):
)
if spiders_module:
print(f"in module:\n {spiders_module.__name__}.{module}")
return Path(spider_file)
def _find_template(self, template: str) -> Path | None:
template_file = Path(self.templates_dir, f"{template}.tmpl")

9
scrapy/core/downloader/contextfactory.py
View File

@ -47,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
instance.
The purpose of this custom class is to provide a ``creatorForNetloc()``
method that returns a ``_ScrapyClientTLSOptions`` instance configured based
on TLS settings provided to the factory.
method that returns:
- a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance
configured based on TLS settings provided to the factory (when the
certificate verification is disabled);
- a result of ``optionsForClientTLS()`` called with those TLS settings
(when the certificate verification is enabled).
"""
def __init__(

6
scrapy/core/downloader/handlers/ftp.py
View File

@ -2,9 +2,9 @@
An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
FTP connection parameters are passed using the request meta field:
- ftp_user (required)
- ftp_password (required)
- ftp_passive (by default, enabled) sets FTP connection passive mode
- ftp_user (optional, falls back to FTP_USER)
- ftp_password (optional, falls back to FTP_PASSWORD)
- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
- ftp_local_filename
- If not given, file data will come in the response.body, as a normal scrapy Response,
which will imply that the entire file will be on memory.

3
scrapy/core/downloader/handlers/http11.py
View File

@ -112,7 +112,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
self._disconnect_timeout: int = 1
async def download_request(self, request: Request) -> Response:
"""Return a deferred for the HTTP download"""
if hasattr(self._crawler.spider, "download_maxsize"): # pragma: no cover
warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
if hasattr(self._crawler.spider, "download_warnsize"): # pragma: no cover
@ -291,7 +290,7 @@ def _tunnel_request_data(
class _TunnelingAgent(Agent):
"""An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
"""An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS
downloads. It may look strange that we have chosen to subclass Agent and not
ProxyAgent but consider that after the tunnel is opened the proxy is
transparent to the client; thus the agent should behave like there is no

6
scrapy/core/http2/agent.py
View File

@ -114,11 +114,7 @@ class H2ConnectionPool:
d.errback(ResponseFailed(errors))
def close_connections(self) -> None:
"""Close all the HTTP/2 connections and remove them from pool
Returns:
Deferred that fires when all connections have been closed
"""
"""Close all the HTTP/2 connections and remove them from pool."""
for conn in self._connections.values():
assert conn.transport is not None # typing
conn.transport.abortConnection()

4
scrapy/core/http2/protocol.py
View File

@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
uri is used to verify that incoming client requests have correct
base URL.
settings -- Scrapy project settings
conn_lost_deferred -- Deferred fires with the reason: Failure to notify
conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify
that connection was lost
tls_verbose_logging -- Whether to log TLS details
"""
@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
def _handle_events(self, events: list[Event]) -> None:
"""Private method which acts as a bridge between the events
received from the HTTP/2 data and IH2EventsHandler
received from the HTTP/2 data and the handlers in this class.
Arguments:
events -- A list of events that the remote peer triggered by sending data

8
scrapy/core/scheduler.py
View File

@ -133,10 +133,10 @@ class Scheduler(BaseScheduler):
(:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by
:attr:`~scrapy.http.Request.priority`.
By default, a single, memory-based priority queue is used for all requests.
When using :setting:`JOBDIR`, a disk-based priority queue is also created,
By default, memory-based priority queues are used for all requests.
When using :setting:`JOBDIR`, disk-based priority queues are also created,
and only unserializable requests are stored in the memory-based priority
queue. For a given priority value, requests in memory take precedence over
queues. For a given priority value, requests in memory take precedence over
requests in disk.
Each priority queue stores requests in separate internal queues, one per
@ -343,7 +343,7 @@ class Scheduler(BaseScheduler):
def open(self, spider: Spider) -> Deferred[None] | None:
"""
(1) initialize the memory queue
(2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
(2) initialize the disk queue if the ``jobdir`` argument wasn't empty
(3) return the result of the dupefilter's ``open`` method
"""
self.spider: Spider = spider

4
scrapy/core/scraper.py
View File

@ -441,7 +441,7 @@ class Scraper:
self, output: Any, response: Response | Failure
) -> Deferred[None]:
"""Process each Request/Item (given in the output parameter) returned
from the given spider.
from the spider.
Items are sent to the item pipelines, requests are scheduled.
"""
@ -451,7 +451,7 @@ class Scraper:
self, output: Any, response: Response | Failure
) -> None:
"""Process each Request/Item (given in the output parameter) returned
from the given spider.
from the spider.
Items are sent to the item pipelines, requests are scheduled.
"""

12
scrapy/crawler.py
View File

@ -569,8 +569,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
"""
Run a crawler with the provided arguments.
It will call the given Crawler's :meth:`~Crawler.crawl` method, while
keeping track of it so it can be stopped later.
It will call the given Crawler's :meth:`~Crawler.crawl_async` method,
while keeping track of it so it can be stopped later.
If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
instance, this method will try to create one using this parameter as
@ -811,7 +811,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
"""
This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS
resolver based on :setting:`DNSCACHE_ENABLED`.
resolver based on :setting:`TWISTED_DNS_RESOLVER`.
If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
@ -913,10 +913,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
When using a reactor it adjusts its pool size to
:setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based
on :setting:`DNSCACHE_ENABLED`.
on :setting:`TWISTED_DNS_RESOLVER`.
If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
If ``stop_after_crawl`` is True, the reactor/event loop will be stopped
after all crawlers have finished, using :meth:`join`.
:param bool stop_after_crawl: stop or not the reactor when all
crawlers have finished

2
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -60,7 +60,7 @@ else:
class HttpCompressionMiddleware:
"""This middleware allows compressed (gzip, deflate) traffic to be
"""This middleware allows compressed (gzip, deflate etc.) traffic to be
sent/received from websites"""
def __init__(

7
scrapy/downloadermiddlewares/redirect.py
View File

@ -196,10 +196,7 @@ class BaseRedirectMiddleware:
class RedirectMiddleware(BaseRedirectMiddleware):
"""
Handle redirection of requests based on response status
and meta-refresh html tag.
"""
"""Handle redirection of requests based on response status."""
@_warn_spider_arg
def process_response(
@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
class MetaRefreshMiddleware(BaseRedirectMiddleware):
"""Handle redirection of requests based on meta-refresh html tag."""
enabled_setting = "METAREFRESH_ENABLED"
def __init__(self, settings: BaseSettings):

8
scrapy/downloadermiddlewares/retry.py
View File

@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error.
You can change the behaviour of this middleware by modifying the scraping settings:
RETRY_TIMES - how many times to retry a failed page
RETRY_HTTP_CODES - which HTTP response codes to retry
Failed pages are collected on the scraping process and rescheduled at the end,
once the spider has finished crawling all regular (non-failed) pages.
"""
from __future__ import annotations
@ -70,8 +67,9 @@ def get_retry_request(
and :ref:`stats <topics-stats>`, and to provide extra logging context (see
:func:`logging.debug`).
*reason* is a string or an :class:`Exception` object that indicates the
reason why the request needs to be retried. It is used to name retry stats.
*reason* is a string, an :class:`Exception` subclass or an
:class:`Exception` object that indicates the reason why the request needs
to be retried. It is used to name retry stats.
*max_retry_times* is a number that determines the maximum number of times
that *request* can be retried. If not specified or ``None``, the number is

2
scrapy/exceptions.py
View File

@ -115,7 +115,7 @@ class UsageError(Exception):
class ScrapyDeprecationWarning(Warning):
"""Warning category for deprecated features, since the default
DeprecationWarning is silenced on Python 2.7+
:exc:`DeprecationWarning` is silenced.
"""

2
scrapy/extensions/logstats.py
View File

@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
class LogStats:
"""Log basic scraping stats periodically like:
* RPM - Requests per Minute
* RPM - Responses per Minute
* IPM - Items per Minute
"""

2
scrapy/extensions/throttle.py
View File

@ -131,7 +131,7 @@ class AutoThrottle:
# It works better with problematic sites.
new_delay = max(target_delay, new_delay)
# Make sure self.mindelay <= new_delay <= self.max_delay
# Make sure self.mindelay <= new_delay <= self.maxdelay
new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
# Dont adjust delay if response status != 200 and new delay is smaller

6
scrapy/http/cookies.py
View File

@ -136,9 +136,9 @@ class _DummyLock:
class WrappedRequest:
"""Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
see http://docs.python.org/library/urllib2.html#urllib2.Request
"""Wraps a :class:`scrapy.Request` class with methods defined by
the :class:`urllib.request.Request` class to interact with
the :class:`http.cookiejar.CookieJar` class.
"""
def __init__(self, request: Request):

5
scrapy/http/response/html.py
View File

@ -1,6 +1,7 @@
"""
This module implements the HtmlResponse class which adds encoding
discovering through HTML encoding declarations to the TextResponse class.
This module implements the :class:`HtmlResponse` class which is used as a
content type marker by :class:`~scrapy.selector.Selector` and can be used in
``isinstance()`` checks.
See documentation in docs/topics/request-response.rst
"""

5
scrapy/http/response/xml.py
View File

@ -1,6 +1,7 @@
"""
This module implements the XmlResponse class which adds encoding
discovering through XML encoding declarations to the TextResponse class.
This module implements the :class:`XmlResponse` class which is used as a
content type marker by :class:`~scrapy.selector.Selector` and can be used in
``isinstance()`` checks.
See documentation in docs/topics/request-response.rst
"""

39
scrapy/item.py
View File

@ -1,7 +1,7 @@
"""
Scrapy Item
See documentation in docs/topics/item.rst
See documentation in docs/topics/items.rst
"""
from __future__ import annotations
@ -25,6 +25,25 @@ class Field(dict[str, Any]):
"""Container of field metadata"""
def _ordered_field_names(cls: type) -> list[str]:
"""Return the names of the :class:`Field` attributes of *cls* in definition
order.
Fields declared in base classes come first, ordered from the topmost base
to the most derived class. Within each class, fields keep their definition
order. A field redefined in a subclass keeps the position of its first
definition.
"""
names: list[str] = []
seen: set[str] = set()
for base in reversed(cls.__mro__):
for name, value in vars(base).items():
if isinstance(value, Field) and name not in seen:
seen.add(name)
names.append(name)
return names
class ItemMeta(ABCMeta):
"""Metaclass_ of :class:`Item` that handles field definitions.
@ -39,13 +58,9 @@ class ItemMeta(ABCMeta):
_class = super().__new__(mcs, "x_" + class_name, new_bases, attrs)
fields = getattr(_class, "fields", {})
new_attrs = {}
for n in dir(_class):
v = getattr(_class, n)
if isinstance(v, Field):
fields[n] = v
elif n in attrs:
new_attrs[n] = attrs[n]
for n in _ordered_field_names(_class):
fields[n] = getattr(_class, n)
new_attrs = {n: v for n, v in attrs.items() if not isinstance(v, Field)}
new_attrs["fields"] = fields
new_attrs["_class"] = _class
@ -80,6 +95,14 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta):
#: those populated. The keys are the field names and the values are the
#: :class:`Field` objects used in the :ref:`Item declaration
#: <topics-items-declaring>`.
#:
#: Fields are kept in definition order: fields declared in base classes
#: come first, followed by fields declared in subclasses, and a field
#: redefined in a subclass keeps the position of its first definition.
#:
#: .. versionchanged:: VERSION
#: Fields are now returned in definition order rather than alphabetical
#: order.
fields: dict[str, Field]
def __init__(self, *args: Any, **kwargs: Any):

4
scrapy/link.py
View File

@ -9,7 +9,9 @@ its documentation in: docs/topics/link-extractors.rst
class Link:
"""Link objects represent an extracted link by the LinkExtractor.
Using the anchor tag sample below to illustrate the parameters::
Using the anchor tag sample below to illustrate the parameters:
.. code-block:: html
<a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>

24
scrapy/logformatter.py
View File

@ -58,18 +58,20 @@ class LogFormatter:
logging an action the method must return ``None``.
Here is an example on how to create a custom log formatter to lower the severity level of
the log message when an item is dropped from the pipeline::
the log message when an item is dropped from the pipeline:
class PoliteLogFormatter(logformatter.LogFormatter):
def dropped(self, item, exception, response, spider):
return {
'level': logging.INFO, # lowering the level from logging.WARNING
'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
'args': {
'exception': exception,
'item': item,
}
}
.. code-block:: python
class PoliteLogFormatter(logformatter.LogFormatter):
def dropped(self, item, exception, response, spider):
return {
"level": logging.INFO, # lowering the level from logging.WARNING
"msg": "Dropped: %(exception)s" + os.linesep + "%(item)s",
"args": {
"exception": exception,
"item": item,
},
}
"""
def crawled(

2
scrapy/mail.py
View File

@ -1,7 +1,5 @@
"""
Mail sending helpers
See documentation in docs/topics/email.rst
"""
from __future__ import annotations

2
scrapy/pipelines/__init__.py
View File

@ -1,7 +1,7 @@
"""
Item pipeline
See documentation in docs/item-pipeline.rst
See documentation in docs/topics/item-pipeline.rst
"""
from __future__ import annotations

2
scrapy/pipelines/files.py
View File

@ -423,7 +423,7 @@ class FTPFilesStore:
class FilesPipeline(MediaPipeline):
"""Abstract pipeline that implement the file downloading
"""Pipeline that implements file downloading.
This pipeline tries to minimize network transfers and file processing,
doing stat of the files and determining if file is new, up-to-date or

2
scrapy/pipelines/images.py
View File

@ -47,7 +47,7 @@ class ImageException(FileException):
class ImagesPipeline(FilesPipeline):
"""Abstract pipeline that implement the image thumbnail generation logic"""
"""Pipeline that implements the handling logic specific to images."""
MEDIA_NAME: str = "image"

7
scrapy/pqueues.py
View File

@ -96,9 +96,10 @@ class ScrapyPriorityQueue:
- The :data:`~scrapy.Request.priority` of the request.
For each combination of the above seen, this class creates an instance of
*downstream_queue_cls* with *key* set to a subdirectory of the persistence
directory, named as the request priority (e.g. ``1``), with an ``s`` suffix
in case of a start request (e.g. ``1s``).
*downstream_queue_cls* (or *start_queue_cls* for start requests if it was
passed) with *key* set to a subdirectory of the persistence directory,
named as the negated request priority (e.g. ``-1``), with an ``s`` suffix
in case of a start request (e.g. ``-1s``).
"""
@classmethod

11
scrapy/settings/__init__.py
View File

@ -445,9 +445,10 @@ class BaseSettings(MutableMapping[str, Any]):
"""
Store a key/value attribute with a given priority.
Settings should be populated *before* configuring the Crawler object
(through the :meth:`~scrapy.crawler.Crawler.configure` method),
otherwise they won't have any effect.
Settings should be populated *before* the Crawler object applies them
(in the :meth:`~scrapy.crawler.Crawler.crawl_async` or
:meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't
have any effect.
:param name: the setting name
:type name: str
@ -604,7 +605,7 @@ class BaseSettings(MutableMapping[str, Any]):
"""
Make a deep copy of current settings.
This method returns a new instance of the :class:`Settings` class,
This method returns a new instance of this class,
populated with the same values and their priorities.
Modifications to the new object won't be reflected on the original
@ -649,7 +650,7 @@ class BaseSettings(MutableMapping[str, Any]):
Make a copy of current settings and convert to a dict.
This method returns a new dict populated with the same values
and their priorities as the current settings.
as the current settings.
Modifications to the returned dict won't be reflected on the original
settings.

5
scrapy/shell.py
View File

@ -75,7 +75,7 @@ if TYPE_CHECKING:
# running event loop.
#
# Side note: it should be possible to remove _request_deferred() by using
# engine.download_async() instead of engine.schedule(), losing the usual stuff
# engine.download_async() instead of engine.crawl(), losing the usual stuff
# like spider middlewares (none of which should be important).
#
# Other architecture problems:
@ -188,7 +188,8 @@ class Shell:
async def _schedule(self, request: Request, spider: Spider | None) -> Response:
"""Send the request to the engine, wait for the result.
Runs in the reactor thread.
Runs in the reactor thread when using the reactor, or in the asyncio
event loop thread otherwise.
"""
if not self.spider:
await self._open_spider(spider)

8
scrapy/spidermiddlewares/base.py
View File

@ -75,7 +75,7 @@ class BaseSpiderMiddleware:
) -> Request | None:
"""Return a processed request from the spider output.
This method is called with a single request from the start seeds or the
This method is called with a single request from ``start()`` or the
spider output. It should return the same or a different request, or
``None`` to ignore it.
@ -84,7 +84,7 @@ class BaseSpiderMiddleware:
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object or ``None`` for
start seeds
start requests
:return: the processed request or ``None``
"""
@ -93,7 +93,7 @@ class BaseSpiderMiddleware:
def get_processed_item(self, item: Any, response: Response | None) -> Any:
"""Return a processed item from the spider output.
This method is called with a single item from the start seeds or the
This method is called with a single item from ``start()`` or the
spider output. It should return the same or a different item, or
``None`` to ignore it.
@ -102,7 +102,7 @@ class BaseSpiderMiddleware:
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object or ``None`` for
start seeds
start items
:return: the processed item or ``None``
"""

2
scrapy/spidermiddlewares/httperror.py
View File

@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)
class HttpError(IgnoreRequest):
"""A non-200 response was filtered"""
"""A non-2xx response was filtered"""
def __init__(self, response: Response, *args: Any, **kwargs: Any):
self.response = response

2
scrapy/spidermiddlewares/referer.py
View File

@ -92,7 +92,7 @@ class ReferrerPolicy(ABC):
)
def origin(self, url: str) -> str | None:
"""Return serialized origin (scheme, host, path) for a request or response URL."""
"""Return serialized origin (scheme, host, port) for a request or response URL."""
return self.strip_url(url, origin_only=True)
def potentially_trustworthy(self, url: str) -> bool:

29
scrapy/spiders/feed.py
View File

@ -54,17 +54,21 @@ class XMLFeedSpider(Spider):
return response
def parse_node(self, response: Response, selector: Selector) -> Any:
"""This method must be overridden with your custom spider functionality"""
"""This method is called for the nodes matching the provided tag name
(itertag). Receives the response and an Selector for each node.
This method must return either an item, a request, or a list
containing any of them.
This method must be overridden with your custom spider functionality.
"""
if hasattr(self, "parse_item"): # backward compatibility
return self.parse_item(response, selector)
raise NotImplementedError
def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any:
"""This method is called for the nodes matching the provided tag name
(itertag). Receives the response and an Selector for each node.
Overriding this method is mandatory. Otherwise, you spider won't work.
This method must return either an item, a request, or a list
containing any of them.
(itertag). Receives the response and an iterable of Selectors.
"""
for selector in nodes:
@ -113,6 +117,9 @@ class CSVFeedSpider(Spider):
It receives a CSV file in a response; iterates through each of its rows,
and calls parse_row with a dict containing each field's data.
This spider also gives the opportunity to override adapt_response and
process_results methods for pre and post-processing purposes.
You can set some options regarding the CSV file, such as the delimiter, quotechar
and the file's headers.
"""
@ -136,16 +143,14 @@ class CSVFeedSpider(Spider):
return response
def parse_row(self, response: Response, row: dict[str, str]) -> Any:
"""This method must be overridden with your custom spider functionality"""
"""Receives a response and a dict (representing each row) with a key for
each provided (or detected) header of the CSV file.
This method must be overridden with your custom spider functionality.
"""
raise NotImplementedError
def parse_rows(self, response: Response) -> Any:
"""Receives a response and a dict (representing each row) with a key for
each provided (or detected) header of the CSV file. This spider also
gives the opportunity to override adapt_response and
process_results methods for pre and post-processing purposes.
"""
for row in csviter(
response, self.delimiter, self.headers, quotechar=self.quotechar
):

2
scrapy/utils/datatypes.py
View File

@ -1,8 +1,6 @@
"""
This module contains data types used by Scrapy which are not included in the
Python Standard Library.
This module must not depend on any module outside the Standard Library.
"""
from __future__ import annotations

52
scrapy/utils/defer.py
View File

@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None:
def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover
warnings.warn(
"scrapy.utils.defer.defer_result() is deprecated, use"
" twisted.internet.defer.success() and twisted.internet.defer.fail(),"
" twisted.internet.defer.succeed() and twisted.internet.defer.fail(),"
" plus an explicit sleep if needed, or explicit reactor.callLater().",
category=ScrapyDeprecationWarning,
stacklevel=2,
@ -469,22 +469,22 @@ def _maybeDeferred_coro(
def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
"""Return an :class:`asyncio.Future` object that wraps *d*.
This function requires
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
installed.
This function requires an installed asyncio reactor or a running asyncio
event loop, see :ref:`using-asyncio`.
When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
callables defined as coroutines <coroutine-support>`, you can only await on
``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
allows you to wait on them::
In this state you cannot await on :class:`~twisted.internet.defer.Deferred`
objects from :ref:`Scrapy callables defined as coroutines
<coroutine-support>`, you can only await on ``Future`` objects. Wrapping
``Deferred`` objects into ``Future`` objects allows you to wait on them:
.. code-block:: python
class MySpider(Spider):
...
async def parse(self, response):
additional_request = scrapy.Request('https://example.org/price')
deferred = self.crawler.engine.download(additional_request)
additional_response = await deferred_to_future(deferred)
deferred = some_dfd_helper()
result = await deferred_to_future(deferred)
.. versionchanged:: 2.14
This function no longer installs an asyncio loop if called before the
@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
in this case.
"""
if not is_asyncio_available():
raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")
raise RuntimeError(
"deferred_to_future() requires an installed asyncio reactor"
" or a running asyncio event loop."
)
return d.asFuture(asyncio.get_event_loop())
@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
defined as a coroutine <coroutine-support>`.
What you can await in Scrapy callables defined as coroutines depends on the
value of :setting:`TWISTED_REACTOR`:
value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`:
- When :ref:`using the asyncio reactor <install-asyncio>`, you can only
await on :class:`asyncio.Future` objects.
- When :ref:`using the asyncio reactor <install-asyncio>`, or :ref:`not
using a reactor at all <asyncio-without-reactor>`, you can only await
on :class:`asyncio.Future` objects.
- When not using the asyncio reactor, you can only await on
:class:`~twisted.internet.defer.Deferred` objects.
- When :ref:`using a non-asyncio reactor <disable-asyncio>`, you can only
await on :class:`~twisted.internet.defer.Deferred` objects.
If you want to write code that uses ``Deferred`` objects but works with any
reactor, use this function on all ``Deferred`` objects::
If you want to write code that uses ``Deferred`` objects but works in both
of these states, use this function on all ``Deferred`` objects:
.. code-block:: python
class MySpider(Spider):
...
async def parse(self, response):
additional_request = scrapy.Request('https://example.org/price')
deferred = self.crawler.engine.download(additional_request)
additional_response = await maybe_deferred_to_future(deferred)
deferred = some_dfd_helper()
result = await maybe_deferred_to_future(deferred)
"""
if not is_asyncio_available():
return d

15
scrapy/utils/deprecate.py
View File

@ -43,15 +43,18 @@ def create_deprecated_class(
It can be used to rename a base class in a library. For example, if we
have
class OldName(SomeClass):
# ...
.. code-block:: python
and we want to rename it to NewName, we can do the following::
class OldName(SomeClass): ...
class NewName(SomeClass):
# ...
and we want to rename it to NewName, we can do the following:
OldName = create_deprecated_class('OldName', NewName)
.. code-block:: python
class NewName(SomeClass): ...
OldName = create_deprecated_class("OldName", NewName)
Then, if user class inherits from OldName, warning is issued. Also, if
some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``

3
scrapy/utils/log.py
View File

@ -248,8 +248,7 @@ def logformatter_adapter(
) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]:
"""
Helper that takes the dictionary output from the methods in LogFormatter
and adapts it into a tuple of positional arguments for logger.log calls,
handling backward compatibility as well.
and adapts it into a tuple of positional arguments for logger.log calls.
"""
level = logkws.get("level", logging.INFO)

2
scrapy/utils/response.py
View File

@ -88,7 +88,7 @@ def open_in_browser(
def parse_details(self, response):
if "item name" not in response.body:
if "item name" not in response.text:
open_in_browser(response)
"""
# circular imports

7
scrapy/utils/signal.py
View File

@ -39,7 +39,7 @@ def send_catch_log(
*arguments: TypingAny,
**named: TypingAny,
) -> list[tuple[TypingAny, TypingAny]]:
"""Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns
"""Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns
Failures instead of exceptions.
"""
dont_log = named.pop("dont_log", ())
@ -172,9 +172,8 @@ async def _send_catch_log_asyncio(
Returns a coroutine that completes once all signal handlers have finished.
This function requires
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
installed.
This function requires an installed asyncio reactor or a running asyncio
event loop.
.. versionadded:: 2.14
"""

9
scrapy/utils/sitemap.py
View File

@ -97,10 +97,11 @@ class Sitemap:
@staticmethod
def _get_tag_name(elem: lxml.etree._Element) -> str:
if TYPE_CHECKING:
assert isinstance(elem.tag, str)
_, _, localname = elem.tag.partition("}")
return localname or elem.tag
tag = elem.tag
if not isinstance(tag, str):
return ""
_, _, localname = tag.partition("}")
return localname or tag
def sitemap_urls_from_robots(

4
scrapy/utils/trackref.py
View File

@ -4,9 +4,7 @@ references to live object instances.
If you want live objects for a particular class to be tracked, you only have to
subclass from object_ref (instead of object).
About performance: This library has a minimal performance impact when enabled,
and no performance penalty at all when disabled (as object_ref becomes just an
alias to object in that case).
This library has a minimal performance impact.
.. note:: PyPy uses a tracing garbage collector, so objects may
remain in the ``live_refs`` longer than expected, even after they

4
scrapy/utils/url.py
View File

@ -117,8 +117,8 @@ def strip_url(
- ``strip_credentials`` removes "user:password@"
- ``strip_default_port`` removes ":80" (resp. ":443", ":21")
from http:// (resp. https://, ftp://) URLs
- ``origin_only`` replaces path component with "/", also dropping
query and fragment components ; it also strips credentials
- ``origin_only`` replaces the path component with "/", also dropping
the query component; it also strips credentials
- ``strip_fragment`` drops any #fragment component
"""

50
tests/test_command_genspider.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import re
import sys
from pathlib import Path
import pytest
@ -9,6 +10,13 @@ from tests.test_commands import TestProjectBase
from tests.utils.cmdline import call, proc
def write_recording_editor(editor: Path) -> None:
"""Create an executable editor script that writes the path it is asked to
open (its last argument) into the file given as its first argument."""
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
editor.chmod(0o755)
def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
@ -63,6 +71,28 @@ class TestGenspiderCommand(TestProjectBase):
assert call("genspider", "--dump=basic", cwd=proj_path) == 0
assert call("genspider", "-d", "basic", cwd=proj_path) == 0
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX shell editor script"
)
def test_edit(self, proj_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
spider = proj_path / self.project_name / "spiders" / "example2.py"
edited = proj_path / "edited.txt"
editor = proj_path / "fake-editor.sh"
write_recording_editor(editor)
# The extra argument exercises shlex-splitting of the EDITOR value.
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
returncode, _, err = proc(
"genspider", "--edit", "example2", "example2.com", cwd=proj_path
)
assert returncode == 0, err
assert "ModuleNotFoundError" not in err
assert spider.exists()
assert (proj_path / edited.read_text(encoding="utf-8")).resolve() == (
spider.resolve()
)
def test_same_name_as_project(self, proj_path: Path) -> None:
assert call("genspider", self.project_name, cwd=proj_path) == 2
assert not (
@ -168,6 +198,26 @@ class TestGenspiderStandaloneCommand:
call("genspider", "example", "example.com", cwd=tmp_path)
assert Path(tmp_path, "example.py").exists()
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX shell editor script"
)
def test_edit(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
spider = tmp_path / "example.py"
edited = tmp_path / "edited.txt"
editor = tmp_path / "fake-editor.sh"
write_recording_editor(editor)
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
returncode, _, err = proc(
"genspider", "--edit", "example", "example.com", cwd=tmp_path
)
assert returncode == 0, err
assert spider.exists()
assert (tmp_path / edited.read_text(encoding="utf-8")).resolve() == (
spider.resolve()
)
@pytest.mark.parametrize("force", [True, False])
def test_same_name_as_existing_file(self, force: bool, tmp_path: Path) -> None:
file_name = "example"

28
tests/test_commands.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import argparse
import json
import sys
from io import StringIO
from shutil import copytree
from typing import TYPE_CHECKING
@ -420,6 +421,33 @@ class TestViewCommand:
assert "URL using the Scrapy downloader and show its" in command.long_desc()
class TestEditCommand(TestProjectBase):
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX shell editor script"
)
def test_edit(self, proj_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
spider = proj_path / self.project_name / "spiders" / "example.py"
edited = proj_path / "edited.txt"
editor = proj_path / "fake-editor.sh"
# Records the file it is asked to open ($2) into the file given as $1.
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
editor.chmod(0o755)
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
assert call("genspider", "example", "example.com", cwd=proj_path) == 0
returncode, _, err = proc("edit", "example", cwd=proj_path)
assert returncode == 0, err
assert (proj_path / edited.read_text(encoding="utf-8")).resolve() == (
spider.resolve()
)
def test_edit_spider_not_found(self, proj_path: Path) -> None:
returncode, _, err = proc("edit", "nonexistent", cwd=proj_path)
assert returncode == 1
assert "Spider not found: nonexistent" in err
class TestHelpMessage(TestProjectBase):
@pytest.mark.parametrize(
"command",

2
tests/test_feedexport.py
View File

@ -746,7 +746,7 @@ class TestFeedExport(TestFeedExportBase):
]
formats = {
"csv": b"baz,egg,foo\r\n,spam1,bar1\r\n",
"csv": b"foo,egg,baz\r\nbar1,spam1,\r\n",
"json": b'[\n{"hello": "world2", "foo": "bar2"}\n]',
"jsonlines": (
b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n'

24
tests/test_item.py
View File

@ -142,6 +142,30 @@ class TestItem:
self.assertSortedEqual(list(item.keys()), ["new"])
self.assertSortedEqual(list(item.values()), ["New"])
def test_fields_order(self):
class TestItem(Item):
name = Field()
keys = Field()
values = Field()
assert list(TestItem.fields) == ["name", "keys", "values"]
def test_fields_order_inheritance(self):
class ParentItem(Item):
name = Field()
keys = Field()
values = Field()
class TestItem(ParentItem):
extra = Field()
keys = Field(serializer=str)
# Inherited fields come first, in their definition order, followed by
# the fields newly defined in the subclass. A redefined field keeps the
# position of its first definition while taking the new metadata.
assert list(TestItem.fields) == ["name", "keys", "values", "extra"]
assert TestItem.fields["keys"] == {"serializer": str}
def test_metaclass_inheritance(self):
class ParentItem(Item):
name = Field()

11
tests/test_utils_sitemap.py
View File

@ -311,3 +311,14 @@ def test_xml_entity_expansion():
"""
)
assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]
def test_sitemap_non_string_tag():
"""With recover=True and resolve_entities=False, libxml2 >= 2.14.6 (used
by lxml >= 6.1.1) preserves undeclared entity reference nodes whose
.tag is a non-string ``Cython function`` object instead of a ``str``.
_get_tag_name must handle this gracefully instead of raising
AttributeError.
"""
results = list(Sitemap(b"<url>&k;"))
assert results == []