From f5b436317033a0c1538382ea8122d32076cc5492 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 16:19:54 +0200 Subject: [PATCH 1/2] Support --curl in commands --- docs/topics/commands.rst | 30 ++++++++++++++++++++++++++++++ scrapy/commands/__init__.py | 22 ++++++++++++++++++++++ scrapy/commands/fetch.py | 29 ++++++++++++++++++++--------- scrapy/commands/parse.py | 25 ++++++++++++++++--------- scrapy/commands/shell.py | 28 +++++++++++++++++++++++++--- tests/test_command_fetch.py | 18 ++++++++++++++++++ tests/test_command_parse.py | 29 +++++++++++++++++++++++++++++ tests/test_command_shell.py | 16 ++++++++++++++++ 8 files changed, 176 insertions(+), 21 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8d1351eb9..a39d5e0e6 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -381,11 +381,21 @@ Supported options: * ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) +* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a + URL argument, taking the URL, HTTP method, headers, cookies and body from the + command (see :meth:`Request.from_curl() `). It cannot + be combined with a URL argument. + + .. versionadded:: VERSION + Usage examples:: $ scrapy fetch --nolog http://www.example.com/some/page.html [ ... html content here ... ] + $ scrapy fetch --nolog --curl 'curl -d title=hello https://httpbin.org/post' + [ ... html content here ... ] + $ scrapy fetch --nolog --headers http://www.example.com/ {'Accept-Ranges': ['bytes'], 'Age': ['1263 '], @@ -415,6 +425,12 @@ Supported options: * ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) +* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a + URL argument (see :meth:`Request.from_curl() `). It + cannot be combined with a URL argument. + + .. versionadded:: VERSION + Usage example:: $ scrapy view http://www.example.com/some/page.html @@ -443,6 +459,12 @@ Supported options: this only affects the URL you may pass as argument on the command line; once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default. +* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a + URL argument (see :meth:`Request.from_curl() `). It + cannot be combined with a URL argument. + + .. versionadded:: VERSION + Usage example:: $ scrapy shell http://www.example.com/some/page.html @@ -481,6 +503,12 @@ Supported options: * ``--callback`` or ``-c``: spider method to use as callback for parsing the response +* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a + URL argument (see :meth:`Request.from_curl() `). It + cannot be combined with a URL argument. + + .. versionadded:: VERSION + * ``--meta`` or ``-m``: additional request meta that will be passed to the callback request. This must be a valid json string. Example: --meta='{"foo" : "bar"}' @@ -669,3 +697,5 @@ The following example adds ``my_command`` command: ], }, ) + +.. _cURL: https://curl.se/ diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 598e8060e..b33861426 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning, UsageError +from scrapy.http import Request from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.python import global_object_name @@ -26,6 +27,27 @@ if TYPE_CHECKING: from scrapy.settings import Settings +def _add_curl_option(parser: argparse.ArgumentParser) -> None: + """Register the ``--curl`` option on commands that accept a URL.""" + parser.add_argument( + "--curl", + metavar="COMMAND", + default=None, + help="build the request from a curl command, e.g. " + "--curl 'curl -d a=1 https://example.com'; cannot be combined " + "with a URL argument", + ) + + +def _request_from_curl(curl_command: str, **kwargs: Any) -> Request: + """Build a :class:`~scrapy.Request` from a curl command, reporting parsing + errors as :exc:`~scrapy.exceptions.UsageError`.""" + try: + return Request.from_curl(curl_command, **kwargs) + except ValueError as e: + raise UsageError(str(e), print_help=False) from e + + class ScrapyCommand(ABC): requires_project: bool = False requires_crawler_process: bool = True diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 0b8311efb..64ebd15c0 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any from w3lib.url import is_url -from scrapy.commands import ScrapyCommand +from scrapy.commands import ScrapyCommand, _add_curl_option, _request_from_curl from scrapy.exceptions import UsageError from scrapy.http import Request, Response from scrapy.utils.datatypes import SequenceExclude @@ -34,6 +34,7 @@ class Command(ScrapyCommand): def add_options(self, parser: ArgumentParser) -> None: super().add_options(parser) + _add_curl_option(parser) parser.add_argument("--spider", dest="spider", help="use this spider") parser.add_argument( "--headers", @@ -67,14 +68,24 @@ class Command(ScrapyCommand): sys.stdout.buffer.write(bytes_ + b"\n") def run(self, args: list[str], opts: Namespace) -> None: - if len(args) != 1 or not is_url(args[0]): - raise UsageError - request = Request( - args[0], - callback=self._print_response, - cb_kwargs={"opts": opts}, - dont_filter=True, - ) + if opts.curl: + if args: + raise UsageError("--curl cannot be combined with a URL argument") + request = _request_from_curl( + opts.curl, + callback=self._print_response, + cb_kwargs={"opts": opts}, + dont_filter=True, + ) + else: + if len(args) != 1 or not is_url(args[0]): + raise UsageError + request = Request( + args[0], + callback=self._print_response, + cb_kwargs={"opts": opts}, + dont_filter=True, + ) # by default, let the framework handle redirects, # i.e. command handles all codes expect 3xx if not opts.no_redirect: diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2ac65bf3f..4e2e012e6 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -10,7 +10,7 @@ from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred from w3lib.url import is_url -from scrapy.commands import BaseRunSpiderCommand +from scrapy.commands import BaseRunSpiderCommand, _add_curl_option, _request_from_curl from scrapy.exceptions import UsageError from scrapy.http import Request, Response from scrapy.utils import display @@ -53,6 +53,7 @@ class Command(BaseRunSpiderCommand): def add_options(self, parser: argparse.ArgumentParser) -> None: super().add_options(parser) + _add_curl_option(parser) parser.add_argument( "--spider", dest="spider", @@ -242,7 +243,7 @@ class Command(BaseRunSpiderCommand): ) return None - def set_spidercls(self, url: str, opts: argparse.Namespace) -> None: + def set_spidercls(self, request: Request, opts: argparse.Namespace) -> None: assert self.crawler_process spider_loader = self.crawler_process.spider_loader if opts.spider: @@ -253,12 +254,12 @@ class Command(BaseRunSpiderCommand): "Unable to find spider: %(spider)s", {"spider": opts.spider} ) else: - self.spidercls = spidercls_for_request(spider_loader, Request(url)) + self.spidercls = spidercls_for_request(spider_loader, request) if not self.spidercls: - logger.error("Unable to find spider for: %(url)s", {"url": url}) + logger.error("Unable to find spider for: %(url)s", {"url": request.url}) async def start(spider: Spider) -> AsyncIterator[Any]: - yield self.prepare_request(spider, Request(url), opts) + yield self.prepare_request(spider, request, opts) if self.spidercls: self.spidercls.start = start # type: ignore[assignment,method-assign] @@ -401,12 +402,18 @@ class Command(BaseRunSpiderCommand): def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments - if not len(args) == 1 or not is_url(args[0]): - raise UsageError - url = args[0] + if opts.curl: + if args: + raise UsageError("--curl cannot be combined with a URL argument") + request = _request_from_curl(opts.curl) + else: + if not len(args) == 1 or not is_url(args[0]): + raise UsageError + request = Request(args[0]) + url = request.url # prepare spidercls - self.set_spidercls(url, opts) + self.set_spidercls(request, opts) if self.spidercls and opts.depth > 0: self.start_parsing(url, opts) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 19138ffd0..926382756 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -10,10 +10,12 @@ import asyncio from threading import Thread from typing import TYPE_CHECKING, Any, ClassVar -from scrapy.commands import ScrapyCommand +from scrapy.commands import ScrapyCommand, _add_curl_option, _request_from_curl from scrapy.crawler import AsyncCrawlerProcess, Crawler +from scrapy.exceptions import UsageError from scrapy.http import Request from scrapy.shell import Shell +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.defer import _schedule_coro from scrapy.utils.spider import DefaultSpider, spidercls_for_request from scrapy.utils.url import guess_scheme @@ -45,6 +47,7 @@ class Command(ScrapyCommand): def add_options(self, parser: ArgumentParser) -> None: super().add_options(parser) + _add_curl_option(parser) parser.add_argument( "-c", dest="code", @@ -66,7 +69,19 @@ class Command(ScrapyCommand): def run(self, args: list[str], opts: Namespace) -> None: url = args[0] if args else None - if url: + request: Request | None = None + if opts.curl: + if url: + raise UsageError("--curl cannot be combined with a URL argument") + request = _request_from_curl(opts.curl, dont_filter=True) + if opts.no_redirect: + request.meta["handle_httpstatus_all"] = True + else: + request.meta["handle_httpstatus_list"] = SequenceExclude( + range(300, 400) + ) + url = request.url + elif url: # first argument may be a local file url = guess_scheme(url) @@ -76,6 +91,10 @@ class Command(ScrapyCommand): spidercls: type[Spider] = DefaultSpider if opts.spider: spidercls = spider_loader.load(opts.spider) + elif request is not None: + spidercls = spidercls_for_request( + spider_loader, request, spidercls, log_multiple=True + ) elif url: spidercls = spidercls_for_request( spider_loader, Request(url), spidercls, log_multiple=True @@ -92,7 +111,10 @@ class Command(ScrapyCommand): self._init_without_reactor(crawler) loop = self._get_reactorless_loop() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code, loop=loop) - shell.start(url=url, redirect=not opts.no_redirect) + if request is not None: + shell.start(request=request, redirect=not opts.no_redirect) + else: + shell.start(url=url, redirect=not opts.no_redirect) def _init_with_reactor(self, crawler: Crawler) -> None: # Create the engine and run start_async() in the main thread diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index d98dac968..df2ee1902 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from typing import TYPE_CHECKING from tests.utils.cmdline import proc @@ -36,3 +37,20 @@ class TestFetchCommand: "fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text") ) assert out.strip() == "Works" + + def test_curl(self, mockserver: MockServer) -> None: + url = mockserver.url("/echo") + _, out, _ = proc("fetch", "--curl", f"curl -d a=1 -H 'X-Test: foo' {url}") + echo = json.loads(out) + assert echo["body"] == "a=1" + assert echo["headers"]["X-Test"] == ["foo"] + + def test_curl_with_url(self, mockserver: MockServer) -> None: + url = mockserver.url("/echo") + code, _, _ = proc("fetch", "--curl", f"curl {url}", url) + assert code != 0 + + def test_curl_invalid(self) -> None: + code, _, err = proc("fetch", "--curl", "not-a-curl-command") + assert code != 0 + assert "curl" in err diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index c210a06e8..5216b2616 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -199,6 +199,35 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} ) assert "DEBUG: It Works!" in stderr + def test_curl(self, proj_path: Path, mockserver: MockServer) -> None: + _, _, stderr = proc( + "parse", + "--spider", + self.spider_name, + "-a", + "test_arg=1", + "--curl", + f"curl {mockserver.url('/html')}", + "-c", + "parse", + "--verbose", + cwd=proj_path, + ) + assert "DEBUG: It Works!" in stderr + + def test_curl_with_url(self, proj_path: Path, mockserver: MockServer) -> None: + url = mockserver.url("/html") + code, _, _ = proc( + "parse", + "--spider", + self.spider_name, + "--curl", + f"curl {url}", + url, + cwd=proj_path, + ) + assert code != 0 + def test_request_with_meta(self, proj_path: Path, mockserver: MockServer) -> None: raw_json_string = '{"foo" : "baz"}' _, _, stderr = proc( diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index f24200f53..513474ad8 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -41,6 +41,22 @@ class TestShellCommand: _, out, _ = proc("shell", mockserver.url("/text"), "-c", "response.body") assert "Works" in out + def test_curl(self, mockserver: MockServer) -> None: + url = mockserver.url("/echo") + _, out, _ = proc( + "shell", + "--curl", + f"curl -d a=1 {url}", + "-c", + "(request.method, request.body)", + ) + assert "('POST', b'a=1')" in out + + def test_curl_with_url(self, mockserver: MockServer) -> None: + url = mockserver.url("/echo") + code, _, _ = proc("shell", "--curl", f"curl {url}", url, "-c", "url") + assert code != 0 + def test_response_type_text(self, mockserver: MockServer) -> None: _, out, _ = proc("shell", mockserver.url("/text"), "-c", "type(response)") assert "TextResponse" in out From 7fc4f13b9dbe20cbf13b2b6e7d89380f7abd713a Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 10 Jul 2026 16:08:01 +0200 Subject: [PATCH 2/2] Sort settings --- docs/topics/components.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c0df86922..f51545513 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -11,8 +11,6 @@ That includes the classes that you may assign to the following settings: - :setting:`ADDONS` -- :setting:`TWISTED_DNS_RESOLVER` - - :setting:`DOWNLOAD_HANDLERS` - :setting:`DOWNLOADER_MIDDLEWARES` @@ -41,6 +39,8 @@ That includes the classes that you may assign to the following settings: - :setting:`SPIDER_MIDDLEWARES` +- :setting:`TWISTED_DNS_RESOLVER` + Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to modify their behavior.