From 98a4c98ef5db8a9a9a0be34d599d1812c283d78b Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 11 Jun 2026 08:39:55 +0200 Subject: [PATCH 1/2] Provide extras for optional dependencies --- .pre-commit-config.yaml | 2 +- docs/conf.py | 1 + docs/intro/install.rst | 48 +++++++++++++++++++++++++++ docs/requirements.in | 2 +- docs/requirements.txt | 2 +- docs/topics/download-handlers.rst | 21 +++++++----- docs/topics/downloader-middleware.rst | 13 ++++---- docs/topics/feed-exports.rst | 18 ++++------ docs/topics/media-pipeline.rst | 24 ++++++-------- docs/topics/shell.rst | 17 ++++------ pyproject.toml | 18 ++++++++++ tox.ini | 2 +- 12 files changed, 113 insertions(+), 55 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b9ef3c04..924d2b5c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.6 + rev: 0.8.7 hooks: - id: sphinx-scrapy diff --git a/docs/conf.py b/docs/conf.py index 99d5df7da..8fa948232 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -158,6 +158,7 @@ scrapy_intersphinx_enable = [ "itemloaders", "parsel", "pytest", + "pypug", "sphinx", "tox", "twisted", diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 8cef04ff1..681308519 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -89,6 +89,54 @@ just like any other Python package. (See :ref:`platform-specific guides ` below for non-Python dependencies that you may need to install beforehand). +.. _extras: + +Optional extras +=============== + +Scrapy provides optional :ref:`extras ` +that install additional dependencies to enable specific features. To install +Scrapy with one or more extras, list them in square brackets: + +.. code-block:: console + + pip install scrapy[s3,images] + +The following extras are available: + +.. list-table:: + :header-rows: 1 + + * - Extra + - Provides + * - ``bpython`` + - :ref:`bpython shell ` + * - ``brotli`` + - :ref:`Brotli response decompression ` + * - ``gcs`` + - :ref:`Google Cloud Storage ` for + :ref:`feed exports ` and + :ref:`media pipelines ` + * - ``http2`` + - :ref:`twisted-http2-handler` + * - ``httpx`` + - :ref:`httpx-handler` + * - ``images`` + - :ref:`Images pipeline ` + * - ``ipython`` + - :ref:`IPython shell ` + * - ``robotparser`` + - :ref:`Robotexclusionrulesparser robots.txt parsing ` + * - ``s3`` + - :ref:`Amazon S3 ` storage for + :ref:`feed exports `, + :ref:`media pipelines `, and + :ref:`S3 downloads ` + * - ``uvloop`` + - `uvloop `_ event loop + * - ``zstd`` + - :ref:`Zstandard response decompression ` + .. _intro-install-platform-notes: diff --git a/docs/requirements.in b/docs/requirements.in index 140791641..04202d98d 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.7 diff --git a/docs/requirements.txt b/docs/requirements.txt index 9c93dacd0..7fa2a383b 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@568cf4279dd1f7da4068fa291c8725bab57cfe98 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 888bfaf08..1f4948e7c 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -169,6 +169,8 @@ of this package for more information. H2DownloadHandler ----------------- +.. note:: Requires the :ref:`http2 ` extra. + .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler | Supported scheme: ``https``. @@ -181,9 +183,6 @@ for them. It's implemented using :mod:`twisted.web.client` and the ``h2`` library. -For this handler to work you need to install the ``Twisted[http2]`` extra -dependency. - If you want to use this handler you need to replace the default one for the ``https`` scheme: @@ -269,9 +268,13 @@ Other limitations: - HTTPS proxies to HTTPS destinations are not supported. +.. _httpx-handler: + HttpxDownloadHandler -------------------- +.. note:: Requires the :ref:`httpx ` extra. + .. versionadded:: 2.15.0 .. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler @@ -284,8 +287,6 @@ HttpxDownloadHandler This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 or HTTP/2 protocol for them. -It's implemented using the ``httpx`` library and needs it to be installed. - If you want to use this handler you need to replace the default ones for the ``http`` and ``https`` schemes: @@ -327,8 +328,8 @@ HTTPX_HTTP2_ENABLED Default: ``False`` -Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra -needs to be installed if you want to enable this setting. +Whether to enable HTTP/2 support in this handler. Requires installing +``httpx[http2]``. .. versionadded:: VERSION @@ -374,9 +375,13 @@ This handler supports ``ftp://host/path`` FTP URIs. It's implemented using :mod:`twisted.protocols.ftp`. +.. _s3-handler: + S3DownloadHandler ----------------- +.. note:: Requires the :ref:`s3 ` extra. + .. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler | Supported scheme: ``s3``. @@ -385,5 +390,3 @@ S3DownloadHandler | :ref:`Requires a Twisted reactor `: no. This handler supports ``s3://bucket/path`` S3 URIs. - -It's implemented using the ``botocore`` library and needs it to be installed. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 8cb29deff..5b7ead050 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -681,6 +681,8 @@ We assume that the spider will not issue Cache-Control directives in requests unless it actually needs them, so directives in requests are not filtered. +.. _http-compression: + HttpCompressionMiddleware ------------------------- @@ -692,14 +694,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is - installed, respectively. + This middleware also supports decoding `brotli-compressed`_ responses with + the :ref:`brotli ` extra, and `zstd-compressed`_ + responses with the :ref:`zstd ` extra. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotli: https://pypi.org/project/Brotli/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings @@ -1201,8 +1201,7 @@ Based on `Robotexclusionrulesparser ` extra. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.RerpRobotParser`` diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6ce5255b9..d6d658b42 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -92,7 +92,6 @@ Marshal - Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal`` - Exporter used: :class:`~scrapy.exporters.MarshalItemExporter` - .. _topics-feed-storage: Storages @@ -106,14 +105,13 @@ The storages backends supported out of the box are: - :ref:`topics-feed-storage-fs` - :ref:`topics-feed-storage-ftp` -- :ref:`topics-feed-storage-s3` (requires boto3_) -- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) +- :ref:`topics-feed-storage-s3` (requires the :ref:`s3 ` extra) +- :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs ` extra) - :ref:`topics-feed-storage-stdout` -Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if the boto3_ -library is installed. - +Some storage backends may be unavailable if the required :ref:`extras ` +are not installed. For example, the S3 backend requires the :ref:`s3 ` +extra. .. _topics-feed-uri-params: @@ -204,7 +202,7 @@ The feeds are stored on `Amazon S3`_. - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` -- Required external libraries: `boto3`_ >= 1.20.0 +- Required extras: :ref:`s3 ` The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -244,7 +242,7 @@ The feeds are stored on `Google Cloud Storage`_. - ``gs://mybucket/path/to/export.csv`` -- Required external libraries: `google-cloud-storage`_. +- Required extras: :ref:`gcs ` For more information about authentication, please refer to `Google Cloud documentation `_. @@ -261,7 +259,6 @@ storage backend is: ``True``. This storage backend uses :ref:`delayed file delivery `. -.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python .. _topics-feed-storage-stdout: @@ -788,6 +785,5 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ -.. _boto3: https://github.com/boto/boto3 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index b542c6c05..1f9c0679c 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -61,6 +61,8 @@ this: Using the Images Pipeline ========================= +.. note:: Requires the :ref:`images ` extra. + Using the :class:`ImagesPipeline` is a lot like using the :class:`FilesPipeline`, except the default field names used are different: you use ``image_urls`` for the image URLs of an item and it will populate an ``images`` field for the information @@ -70,12 +72,6 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for -thumbnailing and normalizing images to JPEG/RGB format. - -.. _Pillow: https://github.com/python-pillow/Pillow - - .. _topics-media-pipeline-enabling: Enabling your Media Pipeline @@ -232,12 +228,13 @@ set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. Amazon S3 storage ----------------- +.. note:: Requires the :ref:`s3 ` extra. + .. setting:: FILES_STORE_S3_ACL .. setting:: IMAGES_STORE_S3_ACL -If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and -:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will -automatically upload the files to the bucket. +:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3 +bucket. Scrapy will automatically upload the files to the bucket. For example, this is a valid :setting:`IMAGES_STORE` value: @@ -272,7 +269,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S AWS_USE_SSL = False # or True (None by default) AWS_VERIFY = False # or True (None by default) -.. _botocore: https://github.com/boto/botocore .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl .. _Minio: https://github.com/minio/minio .. _Zenko CloudServer: https://www.zenko.io/cloudserver/ @@ -283,13 +279,13 @@ For self-hosting you also might feel the need not to use SSL and not to verify S Google Cloud Storage --------------------- +.. note:: Requires the :ref:`gcs ` extra. + .. setting:: FILES_STORE_GCS_ACL .. setting:: IMAGES_STORE_GCS_ACL -:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage -bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ ) - -.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python +:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud +Storage bucket. Scrapy will automatically upload the files to the bucket. For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings: diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 5b826ed18..01d5d8a8f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -17,19 +17,17 @@ spider, without having to run the spider to test every change. Once you get familiarized with the Scrapy shell, you'll see that it's an invaluable tool for developing and debugging your spiders. +.. _shell-config: + Configuring the shell ===================== -If you have `IPython`_ installed, the Scrapy shell will use it (instead of the -standard Python console). The `IPython`_ console is much more powerful and -provides smart auto-completion and colorized output, among other things. +With the :ref:`ipython ` extra, the Scrapy shell will use IPython_ +instead of the :term:`REPL`. IPython provides smart auto-completion, colorized +output, and more. -We highly recommend you install `IPython`_, especially if you're working on -Unix systems (where `IPython`_ excels). See the `IPython installation guide`_ -for more info. - -Scrapy also has support for `bpython`_, and will try to use it where `IPython`_ -is unavailable. +Scrapy also has support for `bpython`_ via the :ref:`bpython ` extra, +and will try to use it where IPython is unavailable. Through Scrapy's settings you can configure it to use any one of ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which @@ -40,7 +38,6 @@ variable; or by defining it in your :ref:`scrapy.cfg `:: shell = bpython .. _IPython: https://ipython.org/ -.. _IPython installation guide: https://ipython.org/install/ .. _bpython: https://bpython-interpreter.org/ Launch the shell diff --git a/pyproject.toml b/pyproject.toml index 2d857dc6d..9e716ff5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,24 @@ Source = "https://github.com/scrapy/scrapy" Tracker = "https://github.com/scrapy/scrapy/issues" "Release notes" = "https://docs.scrapy.org/en/latest/news.html" +[project.optional-dependencies] +bpython = ["bpython>=0.7.1"] +brotli = [ + "brotli>=1.2.0; implementation_name != 'pypy'", + "brotlicffi>=1.2.0.0; implementation_name == 'pypy'", +] +gcs = ["google-cloud-storage>=1.29.0"] +http2 = ["Twisted[http2]>=21.7.0"] +httpx = ["httpx>=0.27.1"] +images = ["Pillow>=8.3.2"] +ipython = ["ipython>=7.1.0"] +robotparser = ["robotexclusionrulesparser>=1.6.2"] +s3 = ["boto3>=1.20.0"] +uvloop = [ + "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'", +] +zstd = ["zstandard>=0.16.0; implementation_name != 'pypy'"] + [project.scripts] scrapy = "scrapy.cmdline:execute" diff --git a/tox.ini b/tox.ini index e21e1f6cf..7347cf01a 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ [tox] requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 + sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.7 envlist = pre-commit,pylint,typing,py,docs minversion = 1.7.0 From 4aa7a0c523f9b0e30b42195683789168b90f2d35 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 10:43:13 +0200 Subject: [PATCH 2/2] Address feedback --- docs/intro/install.rst | 8 +++++--- docs/topics/download-handlers.rst | 17 ++++++++++++----- docs/topics/shell.rst | 18 ++++++++++++------ pyproject.toml | 5 +++-- tox.ini | 4 +++- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 681308519..eeedbce01 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -117,14 +117,14 @@ The following extras are available: - :ref:`Google Cloud Storage ` for :ref:`feed exports ` and :ref:`media pipelines ` - * - ``http2`` - - :ref:`twisted-http2-handler` * - ``httpx`` - - :ref:`httpx-handler` + - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support * - ``images`` - :ref:`Images pipeline ` * - ``ipython`` - :ref:`IPython shell ` + * - ``ptpython`` + - :ref:`ptpython shell ` * - ``robotparser`` - :ref:`Robotexclusionrulesparser robots.txt parsing ` * - ``s3`` @@ -132,6 +132,8 @@ The following extras are available: :ref:`feed exports `, :ref:`media pipelines `, and :ref:`S3 downloads ` + * - ``twisted-http2`` + - :ref:`twisted-http2-handler` * - ``uvloop`` - `uvloop `_ event loop * - ``zstd`` diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index b884f98d9..cd6191bdd 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -173,7 +173,7 @@ of this package for more information. H2DownloadHandler ----------------- -.. note:: Requires the :ref:`http2 ` extra. +.. note:: Requires the :ref:`twisted-http2 ` extra. .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler @@ -291,6 +291,10 @@ HttpxDownloadHandler This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 or HTTP/2 protocol for them. +It's implemented using the httpx_ library. + +.. _httpx: https://www.python-httpx.org/ + If you want to use this handler you need to replace the default ones for the ``http`` and ``https`` schemes: @@ -312,8 +316,8 @@ Features and limitations =========================== ======================================= HTTP proxies Yes -SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) -HTTP/2 Yes (requires ``httpx[http2]``) +SOCKS proxies Yes (SOCKS5) +HTTP/2 Yes ``response.certificate`` DER bytes Per-request ``bindaddress`` No (not supported by the library) TLS implementation Standard library ``ssl`` @@ -332,8 +336,7 @@ HTTPX_HTTP2_ENABLED Default: ``False`` -Whether to enable HTTP/2 support in this handler. Requires installing -``httpx[http2]``. +Whether to enable HTTP/2 support in this handler. .. versionadded:: VERSION @@ -394,3 +397,7 @@ S3DownloadHandler | :ref:`Requires a Twisted reactor `: no. This handler supports ``s3://bucket/path`` S3 URIs. + +It's implemented using the botocore_ library. + +.. _botocore: https://github.com/boto/botocore diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 01d5d8a8f..43672b0b9 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -22,21 +22,27 @@ invaluable tool for developing and debugging your spiders. Configuring the shell ===================== -With the :ref:`ipython ` extra, the Scrapy shell will use IPython_ -instead of the :term:`REPL`. IPython provides smart auto-completion, colorized +With the :ref:`ptpython ` extra, the Scrapy shell will use ptpython_ +instead of the :term:`REPL`. ptpython provides syntax highlighting, smart +auto-completion, and more. + +Failing that, with the :ref:`ipython ` extra, the Scrapy shell will +use IPython_ instead. IPython provides smart auto-completion, colorized output, and more. Scrapy also has support for `bpython`_ via the :ref:`bpython ` extra, -and will try to use it where IPython is unavailable. +and will try to use it where neither ptpython nor IPython is available. Through Scrapy's settings you can configure it to use any one of -``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which -are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment -variable; or by defining it in your :ref:`scrapy.cfg `:: +``ptpython``, ``ipython``, ``bpython`` or the standard ``python`` shell, +regardless of which are installed. This is done by setting the +``SCRAPY_PYTHON_SHELL`` environment variable; or by defining it in your +:ref:`scrapy.cfg `:: [settings] shell = bpython +.. _ptpython: https://github.com/prompt-toolkit/ptpython .. _IPython: https://ipython.org/ .. _bpython: https://bpython-interpreter.org/ diff --git a/pyproject.toml b/pyproject.toml index f62b6b78b..0969486d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,12 +67,13 @@ brotli = [ "brotlicffi>=1.2.0.0; implementation_name == 'pypy'", ] gcs = ["google-cloud-storage>=1.29.0"] -http2 = ["Twisted[http2]>=21.7.0"] -httpx = ["httpx>=0.27.1"] +httpx = ["httpx[http2,socks]>=0.27.1"] images = ["Pillow>=8.3.2"] ipython = ["ipython>=7.1.0"] +ptpython = ["ptpython>=2.0.1"] robotparser = ["robotexclusionrulesparser>=1.6.2"] s3 = ["boto3>=1.20.0"] +twisted-http2 = ["Twisted[http2]>=21.7.0"] uvloop = [ "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'", ] diff --git a/tox.ini b/tox.ini index 5017d7636..6f3226434 100644 --- a/tox.ini +++ b/tox.ini @@ -176,6 +176,7 @@ deps = google-cloud-storage httpx[http2,socks] ipython + ptpython # optional for shell wrapper tests robotexclusionrulesparser uvloop; platform_system != "Windows" and implementation_name != "pypy" zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests @@ -191,8 +192,9 @@ deps = brotli==1.2.0; implementation_name != "pypy" brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 - httpx==0.27.1 + httpx[http2,socks]==0.27.1 ipython==7.1.0 + ptpython==2.0.1 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy"