RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 7fc4f13b9d into b3670369b8

This commit is contained in:
Adrian 2026-07-13 12:51:12 -05:00 committed by GitHub
parent b3670369b8 7fc4f13b9d
commit 83b75d962d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 178 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
docs/topics/commands.rst
View File

@ -381,11 +381,21 @@ Supported options:
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a
URL argument, taking the URL, HTTP method, headers, cookies and body from the
command (see :meth:`Request.from_curl() <scrapy.Request.from_curl>`). It cannot
be combined with a URL argument.
.. versionadded:: VERSION
Usage examples::
$ scrapy fetch --nolog http://www.example.com/some/page.html
[ ... html content here ... ]
$ scrapy fetch --nolog --curl 'curl -d title=hello https://httpbin.org/post'
[ ... html content here ... ]
$ scrapy fetch --nolog --headers http://www.example.com/
{'Accept-Ranges': ['bytes'],
'Age': ['1263 '],
@ -415,6 +425,12 @@ Supported options:
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a
URL argument (see :meth:`Request.from_curl() <scrapy.Request.from_curl>`). It
cannot be combined with a URL argument.
.. versionadded:: VERSION
Usage example::
$ scrapy view http://www.example.com/some/page.html
@ -443,6 +459,12 @@ Supported options:
this only affects the URL you may pass as argument on the command line;
once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default.
* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a
URL argument (see :meth:`Request.from_curl() <scrapy.Request.from_curl>`). It
cannot be combined with a URL argument.
.. versionadded:: VERSION
Usage example::
$ scrapy shell http://www.example.com/some/page.html
@ -481,6 +503,12 @@ Supported options:
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
response
* ``--curl=COMMAND``: build the request from a `cURL`_ command instead of from a
URL argument (see :meth:`Request.from_curl() <scrapy.Request.from_curl>`). It
cannot be combined with a URL argument.
.. versionadded:: VERSION
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
@ -669,3 +697,5 @@ The following example adds ``my_command`` command:
],
},
)
.. _cURL: https://curl.se/

4
docs/topics/components.rst
View File

@ -11,8 +11,6 @@ That includes the classes that you may assign to the following settings:
- :setting:`ADDONS`
- :setting:`TWISTED_DNS_RESOLVER`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_MIDDLEWARES`
@ -41,6 +39,8 @@ That includes the classes that you may assign to the following settings:
- :setting:`SPIDER_MIDDLEWARES`
- :setting:`TWISTED_DNS_RESOLVER`
Third-party Scrapy components may also let you define additional Scrapy
components, usually configurable through :ref:`settings <topics-settings>`, to
modify their behavior.

22
scrapy/commands/__init__.py
View File

@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from twisted.python import failure
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.http import Request
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.python import global_object_name
@ -26,6 +27,27 @@ if TYPE_CHECKING:
from scrapy.settings import Settings
def _add_curl_option(parser: argparse.ArgumentParser) -> None:
"""Register the ``--curl`` option on commands that accept a URL."""
parser.add_argument(
"--curl",
metavar="COMMAND",
default=None,
help="build the request from a curl command, e.g. "
"--curl 'curl -d a=1 https://example.com'; cannot be combined "
"with a URL argument",
)
def _request_from_curl(curl_command: str, **kwargs: Any) -> Request:
"""Build a :class:`~scrapy.Request` from a curl command, reporting parsing
errors as :exc:`~scrapy.exceptions.UsageError`."""
try:
return Request.from_curl(curl_command, **kwargs)
except ValueError as e:
raise UsageError(str(e), print_help=False) from e
class ScrapyCommand(ABC):
requires_project: bool = False
requires_crawler_process: bool = True

29
scrapy/commands/fetch.py
View File

@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
from scrapy.commands import ScrapyCommand, _add_curl_option, _request_from_curl
from scrapy.exceptions import UsageError
from scrapy.http import Request, Response
from scrapy.utils.datatypes import SequenceExclude
@ -34,6 +34,7 @@ class Command(ScrapyCommand):
def add_options(self, parser: ArgumentParser) -> None:
super().add_options(parser)
_add_curl_option(parser)
parser.add_argument("--spider", dest="spider", help="use this spider")
parser.add_argument(
"--headers",
@ -67,14 +68,24 @@ class Command(ScrapyCommand):
sys.stdout.buffer.write(bytes_ + b"\n")
def run(self, args: list[str], opts: Namespace) -> None:
if len(args) != 1 or not is_url(args[0]):
raise UsageError
request = Request(
args[0],
callback=self._print_response,
cb_kwargs={"opts": opts},
dont_filter=True,
)
if opts.curl:
if args:
raise UsageError("--curl cannot be combined with a URL argument")
request = _request_from_curl(
opts.curl,
callback=self._print_response,
cb_kwargs={"opts": opts},
dont_filter=True,
)
else:
if len(args) != 1 or not is_url(args[0]):
raise UsageError
request = Request(
args[0],
callback=self._print_response,
cb_kwargs={"opts": opts},
dont_filter=True,
)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:

25
scrapy/commands/parse.py
View File

@ -10,7 +10,7 @@ from itemadapter import ItemAdapter
from twisted.internet.defer import Deferred, maybeDeferred
from w3lib.url import is_url
from scrapy.commands import BaseRunSpiderCommand
from scrapy.commands import BaseRunSpiderCommand, _add_curl_option, _request_from_curl
from scrapy.exceptions import UsageError
from scrapy.http import Request, Response
from scrapy.utils import display
@ -53,6 +53,7 @@ class Command(BaseRunSpiderCommand):
def add_options(self, parser: argparse.ArgumentParser) -> None:
super().add_options(parser)
_add_curl_option(parser)
parser.add_argument(
"--spider",
dest="spider",
@ -242,7 +243,7 @@ class Command(BaseRunSpiderCommand):
)
return None
def set_spidercls(self, url: str, opts: argparse.Namespace) -> None:
def set_spidercls(self, request: Request, opts: argparse.Namespace) -> None:
assert self.crawler_process
spider_loader = self.crawler_process.spider_loader
if opts.spider:
@ -253,12 +254,12 @@ class Command(BaseRunSpiderCommand):
"Unable to find spider: %(spider)s", {"spider": opts.spider}
)
else:
self.spidercls = spidercls_for_request(spider_loader, Request(url))
self.spidercls = spidercls_for_request(spider_loader, request)
if not self.spidercls:
logger.error("Unable to find spider for: %(url)s", {"url": url})
logger.error("Unable to find spider for: %(url)s", {"url": request.url})
async def start(spider: Spider) -> AsyncIterator[Any]:
yield self.prepare_request(spider, Request(url), opts)
yield self.prepare_request(spider, request, opts)
if self.spidercls:
self.spidercls.start = start # type: ignore[assignment,method-assign]
@ -401,12 +402,18 @@ class Command(BaseRunSpiderCommand):
def run(self, args: list[str], opts: argparse.Namespace) -> None:
# parse arguments
if not len(args) == 1 or not is_url(args[0]):
raise UsageError
url = args[0]
if opts.curl:
if args:
raise UsageError("--curl cannot be combined with a URL argument")
request = _request_from_curl(opts.curl)
else:
if not len(args) == 1 or not is_url(args[0]):
raise UsageError
request = Request(args[0])
url = request.url
# prepare spidercls
self.set_spidercls(url, opts)
self.set_spidercls(request, opts)
if self.spidercls and opts.depth > 0:
self.start_parsing(url, opts)

28
scrapy/commands/shell.py
View File

@ -10,10 +10,12 @@ import asyncio
from threading import Thread
from typing import TYPE_CHECKING, Any, ClassVar
from scrapy.commands import ScrapyCommand
from scrapy.commands import ScrapyCommand, _add_curl_option, _request_from_curl
from scrapy.crawler import AsyncCrawlerProcess, Crawler
from scrapy.exceptions import UsageError
from scrapy.http import Request
from scrapy.shell import Shell
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import _schedule_coro
from scrapy.utils.spider import DefaultSpider, spidercls_for_request
from scrapy.utils.url import guess_scheme
@ -45,6 +47,7 @@ class Command(ScrapyCommand):
def add_options(self, parser: ArgumentParser) -> None:
super().add_options(parser)
_add_curl_option(parser)
parser.add_argument(
"-c",
dest="code",
@ -66,7 +69,19 @@ class Command(ScrapyCommand):
def run(self, args: list[str], opts: Namespace) -> None:
url = args[0] if args else None
if url:
request: Request | None = None
if opts.curl:
if url:
raise UsageError("--curl cannot be combined with a URL argument")
request = _request_from_curl(opts.curl, dont_filter=True)
if opts.no_redirect:
request.meta["handle_httpstatus_all"] = True
else:
request.meta["handle_httpstatus_list"] = SequenceExclude(
range(300, 400)
)
url = request.url
elif url:
# first argument may be a local file
url = guess_scheme(url)
@ -76,6 +91,10 @@ class Command(ScrapyCommand):
spidercls: type[Spider] = DefaultSpider
if opts.spider:
spidercls = spider_loader.load(opts.spider)
elif request is not None:
spidercls = spidercls_for_request(
spider_loader, request, spidercls, log_multiple=True
)
elif url:
spidercls = spidercls_for_request(
spider_loader, Request(url), spidercls, log_multiple=True
@ -92,7 +111,10 @@ class Command(ScrapyCommand):
self._init_without_reactor(crawler)
loop = self._get_reactorless_loop()
shell = Shell(crawler, update_vars=self.update_vars, code=opts.code, loop=loop)
shell.start(url=url, redirect=not opts.no_redirect)
if request is not None:
shell.start(request=request, redirect=not opts.no_redirect)
else:
shell.start(url=url, redirect=not opts.no_redirect)
def _init_with_reactor(self, crawler: Crawler) -> None:
# Create the engine and run start_async() in the main thread

18
tests/test_command_fetch.py
View File

@ -1,5 +1,6 @@
from __future__ import annotations
import json
from typing import TYPE_CHECKING
from tests.utils.cmdline import proc
@ -36,3 +37,20 @@ class TestFetchCommand:
"fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text")
)
assert out.strip() == "Works"
def test_curl(self, mockserver: MockServer) -> None:
url = mockserver.url("/echo")
_, out, _ = proc("fetch", "--curl", f"curl -d a=1 -H 'X-Test: foo' {url}")
echo = json.loads(out)
assert echo["body"] == "a=1"
assert echo["headers"]["X-Test"] == ["foo"]
def test_curl_with_url(self, mockserver: MockServer) -> None:
url = mockserver.url("/echo")
code, _, _ = proc("fetch", "--curl", f"curl {url}", url)
assert code != 0
def test_curl_invalid(self) -> None:
code, _, err = proc("fetch", "--curl", "not-a-curl-command")
assert code != 0
assert "curl" in err

29
tests/test_command_parse.py
View File

@ -199,6 +199,35 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
)
assert "DEBUG: It Works!" in stderr
def test_curl(self, proj_path: Path, mockserver: MockServer) -> None:
_, _, stderr = proc(
"parse",
"--spider",
self.spider_name,
"-a",
"test_arg=1",
"--curl",
f"curl {mockserver.url('/html')}",
"-c",
"parse",
"--verbose",
cwd=proj_path,
)
assert "DEBUG: It Works!" in stderr
def test_curl_with_url(self, proj_path: Path, mockserver: MockServer) -> None:
url = mockserver.url("/html")
code, _, _ = proc(
"parse",
"--spider",
self.spider_name,
"--curl",
f"curl {url}",
url,
cwd=proj_path,
)
assert code != 0
def test_request_with_meta(self, proj_path: Path, mockserver: MockServer) -> None:
raw_json_string = '{"foo" : "baz"}'
_, _, stderr = proc(

16
tests/test_command_shell.py
View File

@ -41,6 +41,22 @@ class TestShellCommand:
_, out, _ = proc("shell", mockserver.url("/text"), "-c", "response.body")
assert "Works" in out
def test_curl(self, mockserver: MockServer) -> None:
url = mockserver.url("/echo")
_, out, _ = proc(
"shell",
"--curl",
f"curl -d a=1 {url}",
"-c",
"(request.method, request.body)",
)
assert "('POST', b'a=1')" in out
def test_curl_with_url(self, mockserver: MockServer) -> None:
url = mockserver.url("/echo")
code, _, _ = proc("shell", "--curl", f"curl {url}", url, "-c", "url")
assert code != 0
def test_response_type_text(self, mockserver: MockServer) -> None:
_, out, _ = proc("shell", mockserver.url("/text"), "-c", "type(response)")
assert "TextResponse" in out