From 4aa7a0c523f9b0e30b42195683789168b90f2d35 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 10:43:13 +0200 Subject: [PATCH] Address feedback --- docs/intro/install.rst | 8 +++++--- docs/topics/download-handlers.rst | 17 ++++++++++++----- docs/topics/shell.rst | 18 ++++++++++++------ pyproject.toml | 5 +++-- tox.ini | 4 +++- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 681308519..eeedbce01 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -117,14 +117,14 @@ The following extras are available: - :ref:`Google Cloud Storage ` for :ref:`feed exports ` and :ref:`media pipelines ` - * - ``http2`` - - :ref:`twisted-http2-handler` * - ``httpx`` - - :ref:`httpx-handler` + - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support * - ``images`` - :ref:`Images pipeline ` * - ``ipython`` - :ref:`IPython shell ` + * - ``ptpython`` + - :ref:`ptpython shell ` * - ``robotparser`` - :ref:`Robotexclusionrulesparser robots.txt parsing ` * - ``s3`` @@ -132,6 +132,8 @@ The following extras are available: :ref:`feed exports `, :ref:`media pipelines `, and :ref:`S3 downloads ` + * - ``twisted-http2`` + - :ref:`twisted-http2-handler` * - ``uvloop`` - `uvloop `_ event loop * - ``zstd`` diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index b884f98d9..cd6191bdd 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -173,7 +173,7 @@ of this package for more information. H2DownloadHandler ----------------- -.. note:: Requires the :ref:`http2 ` extra. +.. note:: Requires the :ref:`twisted-http2 ` extra. .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler @@ -291,6 +291,10 @@ HttpxDownloadHandler This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 or HTTP/2 protocol for them. +It's implemented using the httpx_ library. + +.. _httpx: https://www.python-httpx.org/ + If you want to use this handler you need to replace the default ones for the ``http`` and ``https`` schemes: @@ -312,8 +316,8 @@ Features and limitations =========================== ======================================= HTTP proxies Yes -SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) -HTTP/2 Yes (requires ``httpx[http2]``) +SOCKS proxies Yes (SOCKS5) +HTTP/2 Yes ``response.certificate`` DER bytes Per-request ``bindaddress`` No (not supported by the library) TLS implementation Standard library ``ssl`` @@ -332,8 +336,7 @@ HTTPX_HTTP2_ENABLED Default: ``False`` -Whether to enable HTTP/2 support in this handler. Requires installing -``httpx[http2]``. +Whether to enable HTTP/2 support in this handler. .. versionadded:: VERSION @@ -394,3 +397,7 @@ S3DownloadHandler | :ref:`Requires a Twisted reactor `: no. This handler supports ``s3://bucket/path`` S3 URIs. + +It's implemented using the botocore_ library. + +.. _botocore: https://github.com/boto/botocore diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 01d5d8a8f..43672b0b9 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -22,21 +22,27 @@ invaluable tool for developing and debugging your spiders. Configuring the shell ===================== -With the :ref:`ipython ` extra, the Scrapy shell will use IPython_ -instead of the :term:`REPL`. IPython provides smart auto-completion, colorized +With the :ref:`ptpython ` extra, the Scrapy shell will use ptpython_ +instead of the :term:`REPL`. ptpython provides syntax highlighting, smart +auto-completion, and more. + +Failing that, with the :ref:`ipython ` extra, the Scrapy shell will +use IPython_ instead. IPython provides smart auto-completion, colorized output, and more. Scrapy also has support for `bpython`_ via the :ref:`bpython ` extra, -and will try to use it where IPython is unavailable. +and will try to use it where neither ptpython nor IPython is available. Through Scrapy's settings you can configure it to use any one of -``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which -are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment -variable; or by defining it in your :ref:`scrapy.cfg `:: +``ptpython``, ``ipython``, ``bpython`` or the standard ``python`` shell, +regardless of which are installed. This is done by setting the +``SCRAPY_PYTHON_SHELL`` environment variable; or by defining it in your +:ref:`scrapy.cfg `:: [settings] shell = bpython +.. _ptpython: https://github.com/prompt-toolkit/ptpython .. _IPython: https://ipython.org/ .. _bpython: https://bpython-interpreter.org/ diff --git a/pyproject.toml b/pyproject.toml index f62b6b78b..0969486d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,12 +67,13 @@ brotli = [ "brotlicffi>=1.2.0.0; implementation_name == 'pypy'", ] gcs = ["google-cloud-storage>=1.29.0"] -http2 = ["Twisted[http2]>=21.7.0"] -httpx = ["httpx>=0.27.1"] +httpx = ["httpx[http2,socks]>=0.27.1"] images = ["Pillow>=8.3.2"] ipython = ["ipython>=7.1.0"] +ptpython = ["ptpython>=2.0.1"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] +twisted-http2 = ["Twisted[http2]>=21.7.0"] uvloop = [ "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'", ] diff --git a/tox.ini b/tox.ini index 5017d7636..6f3226434 100644 --- a/tox.ini +++ b/tox.ini @@ -176,6 +176,7 @@ deps = google-cloud-storage httpx[http2,socks] ipython + ptpython # optional for shell wrapper tests robotexclusionrulesparser uvloop; platform_system != "Windows" and implementation_name != "pypy" zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests @@ -191,8 +192,9 @@ deps = brotli==1.2.0; implementation_name != "pypy" brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 - httpx==0.27.1 + httpx[http2,socks]==0.27.1 ipython==7.1.0 + ptpython==2.0.1 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy"