From f4d1905cca7b784f88bff00f6c43ed7e95ebeaa0 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 14 Jul 2026 14:51:39 +0500 Subject: [PATCH 1/3] Modernize some code in examples. --- docs/topics/developer-tools.rst | 3 +-- docs/topics/request-response.rst | 2 +- docs/topics/settings.rst | 4 ++-- docs/topics/signals.rst | 4 ++-- docs/topics/spiders.rst | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 05dffcdda..a5ffe00f1 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -246,7 +246,6 @@ also request each page to get every quote on the site: .. code-block:: python import scrapy - import json class QuoteSpider(scrapy.Spider): @@ -256,7 +255,7 @@ also request each page to get every quote on the site: start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): - data = json.loads(response.text) + data = response.json() for quote in data["quotes"]: yield {"quote": quote["text"]} if data["has_next"]: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a5f799444..8ce827676 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -436,7 +436,7 @@ errors if needed: ) def parse_httpbin(self, response): - self.logger.info("Got successful response from {}".format(response.url)) + self.logger.info(f"Got successful response from {response.url}") # do something useful here... def errback_httpbin(self, failure): diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a592f8bc5..75e213fc0 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2209,7 +2209,7 @@ In order to use the reactor installed by Scrapy: def __init__(self, *args, **kwargs): self.timeout = int(kwargs.pop("timeout", "60")) - super(QuotesSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) async def start(self): reactor.callLater(self.timeout, self.stop) @@ -2238,7 +2238,7 @@ which raises an exception, becomes: def __init__(self, *args, **kwargs): self.timeout = int(kwargs.pop("timeout", "60")) - super(QuotesSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) async def start(self): from twisted.internet import reactor diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 531232149..03996bee6 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -34,7 +34,7 @@ Here is a simple example showing how you can catch signals and perform some acti @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs) + spider = super().from_crawler(crawler, *args, **kwargs) crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed) return spider @@ -72,7 +72,7 @@ Let's take an example using :ref:`coroutines `: @classmethod def from_crawler(cls, crawler, *args, **kwargs): - spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs) + spider = super().from_crawler(crawler, *args, **kwargs) crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped) return spider diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 887da0a15..af5c9a565 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -314,7 +314,7 @@ Spiders can access arguments in their `__init__` methods: name = "myspider" def __init__(self, category=None, *args, **kwargs): - super(MySpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.start_urls = [f"http://www.example.com/categories/{category}"] # ... From 7f69880504e07d22e48bd71b1ecc5921f97e91f4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 14 Jul 2026 14:59:26 +0500 Subject: [PATCH 2/3] Remove docs for the non-existent MEMDEBUG_NOTIFY. --- docs/topics/settings.rst | 15 --------------- scrapy/settings/default_settings.py | 2 -- 2 files changed, 17 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 75e213fc0..4bf3780bf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1644,21 +1644,6 @@ Default: ``False`` Whether to enable memory debugging. -.. setting:: MEMDEBUG_NOTIFY - -MEMDEBUG_NOTIFY ---------------- - -Default: ``[]`` - -When memory debugging is enabled a memory report will be sent to the specified -addresses if this setting is not empty, otherwise the report will be written to -the log. - -Example:: - - MEMDEBUG_NOTIFY = ['user@example.com'] - .. setting:: MEMUSAGE_ENABLED MEMUSAGE_ENABLED diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3a7dfd018..d283336ab 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -154,7 +154,6 @@ __all__ = [ "MAIL_TLS", "MAIL_USER", "MEMDEBUG_ENABLED", - "MEMDEBUG_NOTIFY", "MEMUSAGE_CHECK_INTERVAL_SECONDS", "MEMUSAGE_ENABLED", "MEMUSAGE_LIMIT_MB", @@ -470,7 +469,6 @@ MAIL_SSL = False MAIL_TLS = False MEMDEBUG_ENABLED = False # enable memory debugging -MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown MEMUSAGE_ENABLED = True MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0 From 4837e8afde8463e79d715896b82c34b6a0392504 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 14 Jul 2026 15:20:39 +0500 Subject: [PATCH 3/3] Add more code-block markup. --- docs/topics/addons.rst | 12 ++++++-- docs/topics/exporters.rst | 44 ++++++++++++++++++++--------- docs/topics/feed-exports.rst | 48 +++++++++++++++++--------------- docs/topics/leaks.rst | 11 ++++++-- docs/topics/link-extractors.rst | 4 ++- docs/topics/loaders.rst | 4 ++- docs/topics/media-pipeline.rst | 8 ++++-- docs/topics/request-response.rst | 36 ++++++++++++++++++------ docs/topics/settings.rst | 14 +++++++--- docs/topics/shell.rst | 4 ++- docs/topics/spiders.rst | 31 +++++++++++++++------ 11 files changed, 148 insertions(+), 68 deletions(-) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 78b1ab92b..75f15b3ae 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -21,10 +21,14 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. This is an example where two add-ons are enabled in a project's -``settings.py``:: +``settings.py``: + +.. skip: next + +.. code-block:: python ADDONS = { - 'path.to.someaddon': 0, + "path.to.someaddon": 0, SomeAddonClass: 1, } @@ -56,7 +60,9 @@ the following methods: :type settings: :class:`~scrapy.settings.BaseSettings` The settings set by the add-on should use the ``addon`` priority (see -:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`): + +.. code-block:: python class MyAddon: def update_settings(self, settings): diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index f0b704b9d..ecd154122 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -211,13 +211,17 @@ BaseItemExporter - ``None`` (all fields [2]_, default) - - A list of fields:: + - A list of fields: - ['field1', 'field2'] + .. code-block:: python - - A dict where keys are fields and values are output names:: + ["field1", "field2"] - {'field1': 'Field 1', 'field2': 'Field 2'} + - A dict where keys are fields and values are output names: + + .. code-block:: python + + {"field1": "Field 1", "field2": "Field 2"} .. [1] Not all exporters respect the specified field order. .. [2] When using :ref:`item objects ` that do not expose @@ -273,7 +277,9 @@ XmlItemExporter The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method. - A typical output of this exporter would be:: + A typical output of this exporter would be: + + .. code-block:: xml @@ -291,11 +297,17 @@ XmlItemExporter exported by serializing each value inside a ```` element. This is for convenience, as multi-valued fields are very common. - For example, the item:: + For example, the item: - Item(name=['John', 'Doe'], age='23') + .. skip: next - Would be serialized as:: + .. code-block:: python + + Item(name=["John", "Doe"], age="23") + + Would be serialized as: + + .. code-block:: xml @@ -379,10 +391,12 @@ PprintItemExporter The additional keyword arguments of this ``__init__`` method are passed to the :class:`BaseItemExporter` ``__init__`` method. - A typical output of this exporter would be:: + A typical output of this exporter would be: - {'name': 'Color TV', 'price': '1200'} - {'name': 'DVD player', 'price': '200'} + .. code-block:: python + + {"name": "Color TV", "price": "1200"} + {"name": "DVD player", "price": "200"} Longer lines (when present) are pretty-formatted. @@ -400,7 +414,9 @@ JsonItemExporter :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) - A typical output of this exporter would be:: + A typical output of this exporter would be: + + .. code-block:: json [{"name": "Color TV", "price": "1200"}, {"name": "DVD player", "price": "200"}] @@ -429,7 +445,9 @@ JsonLinesItemExporter :param file: the file-like object to use for exporting the data. Its ``write`` method should accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc) - A typical output of this exporter would be:: + A typical output of this exporter would be: + + .. code-block:: json {"name": "Color TV", "price": "1200"} {"name": "DVD player", "price": "200"} diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 1bab3bfc2..51e142b04 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -434,33 +434,37 @@ This setting is required for enabling the feed export feature. See :ref:`topics-feed-storage-backends` for supported URI schemes. -For instance:: +For instance: + +.. skip: next + +.. code-block:: python { - 'items.json': { - 'format': 'json', - 'encoding': 'utf8', - 'store_empty': False, - 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'], - 'fields': None, - 'indent': 4, - 'item_export_kwargs': { - 'export_empty_fields': True, + "items.json": { + "format": "json", + "encoding": "utf8", + "store_empty": False, + "item_classes": [MyItemClass1, "myproject.items.MyItemClass2"], + "fields": None, + "indent": 4, + "item_export_kwargs": { + "export_empty_fields": True, }, }, - '/home/user/documents/items.xml': { - 'format': 'xml', - 'fields': ['name', 'price'], - 'item_filter': MyCustomFilter1, - 'encoding': 'latin1', - 'indent': 8, + "/home/user/documents/items.xml": { + "format": "xml", + "fields": ["name", "price"], + "item_filter": MyCustomFilter1, + "encoding": "latin1", + "indent": 8, }, - pathlib.Path('items.csv.gz'): { - 'format': 'csv', - 'fields': ['price', 'name'], - 'item_filter': 'myproject.filters.MyCustomFilter2', - 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'], - 'gzip_compresslevel': 5, + pathlib.Path("items.csv.gz"): { + "format": "csv", + "fields": ["price", "name"], + "item_filter": "myproject.filters.MyCustomFilter2", + "postprocessing": [MyPlugin1, "scrapy.extensions.postprocessing.GzipPlugin"], + "gzip_compresslevel": 5, }, } diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 3337cc5d0..d4577601d 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -106,10 +106,15 @@ A real example -------------- Let's see a concrete example of a hypothetical case of memory leaks. -Suppose we have some spider with a line similar to this one:: +Suppose we have some spider with a line similar to this one: - return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}", - callback=self.parse, cb_kwargs={'referer': response}) +.. code-block:: python + + return Request( + f"http://www.somenastyspider.com/product.php?pid={product_id}", + callback=self.parse, + cb_kwargs={"referer": response}, + ) That line is passing a response reference inside a request which effectively ties the response lifetime to the requests' one, and that would definitely diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 3fc896507..c1d872dd0 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -36,7 +36,9 @@ Link extractor reference The link extractor class is :class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it -can also be imported as ``scrapy.linkextractors.LinkExtractor``:: +can also be imported as ``scrapy.linkextractors.LinkExtractor``: + +.. code-block:: python from scrapy.linkextractors import LinkExtractor diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 765e7f6d7..314e2c1ae 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -350,7 +350,9 @@ When parsing related values from a subsection of a document, it can be useful to create nested loaders. Imagine you're extracting details from a footer of a page that looks something like: -Example:: +Example: + +.. code-block:: html