From 76eba9977bc7a8388633e6ba6c856f90eb738d41 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 14 Feb 2023 13:40:38 +0500 Subject: [PATCH] Skip or fix failing code block tests (#5826) --- docs/faq.rst | 6 +++++- docs/intro/tutorial.rst | 4 ++++ docs/topics/contracts.rst | 1 + docs/topics/coroutines.rst | 1 + docs/topics/debug.rst | 1 + docs/topics/downloader-middleware.rst | 1 + docs/topics/email.rst | 2 ++ docs/topics/exporters.rst | 1 + docs/topics/item-pipeline.rst | 1 + docs/topics/link-extractors.rst | 2 +- docs/topics/media-pipeline.rst | 4 ++++ docs/topics/practices.rst | 4 ++++ docs/topics/request-response.rst | 6 ++++++ docs/topics/settings.rst | 5 +++++ docs/topics/signals.rst | 3 +++ docs/topics/spider-middleware.rst | 3 +++ docs/topics/spiders.rst | 11 +++++++++-- 17 files changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 241a588a8..031f4b942 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -37,6 +37,7 @@ and extract whatever data you need from it. Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser: +.. skip: next .. code-block:: python from bs4 import BeautifulSoup @@ -239,6 +240,9 @@ higher) in your spider: .. code-block:: python + from scrapy.spiders import CrawlSpider + + class MySpider(CrawlSpider): name = "myspider" @@ -416,4 +420,4 @@ See :issue:`2680`. .. _user agents: https://en.wikipedia.org/wiki/User_agent .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) .. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search \ No newline at end of file +.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 5904482a5..064ce05f8 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -664,6 +664,8 @@ Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no need to call urljoin. Note that ``response.follow`` just returns a Request instance; you still have to yield this Request. +.. skip: start + You can also pass a selector to ``response.follow`` instead of a string; this selector should extract necessary attributes: @@ -694,6 +696,8 @@ or, shortening it further: yield from response.follow_all(css="ul.pager a", callback=self.parse) +.. skip: end + More examples and patterns -------------------------- diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 211a0f5f2..2d61026e9 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -118,6 +118,7 @@ Raise :class:`~scrapy.exceptions.ContractFail` from Here is a demo contract which checks the presence of a custom header in the response received: +.. skip: next .. code-block:: python from scrapy.contracts import Contract diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a0c005204..3916bd295 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -95,6 +95,7 @@ coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. This means you can use many useful Python libraries providing such code: +.. skip: next .. code-block:: python class MySpiderDeferred(Spider): diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 6ecba110c..49c5b0410 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -7,6 +7,7 @@ Debugging Spiders This document explains the most common techniques for debugging spiders. Consider the following Scrapy spider below: +.. skip: next .. code-block:: python import scrapy diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a7957dd3f..7665a901a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -232,6 +232,7 @@ There is support for keeping multiple cookie sessions per spider by using the For example: +.. skip: next .. code-block:: python for i, url in enumerate(urls): diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 62a5a65bd..d6a7ad354 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -30,6 +30,7 @@ the standard ``__init__`` method: Or you can instantiate it passing a Scrapy settings object, which will respect the :ref:`settings `: +.. skip: start .. code-block:: python mailer = MailSender.from_settings(settings) @@ -44,6 +45,7 @@ And here is how to use it to send an e-mail (without attachments): body="Some body", cc=["another@example.com"], ) +.. skip: end MailSender class reference ========================== diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index d09a6274f..7a85c099b 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -143,6 +143,7 @@ Built-in Item Exporters reference Here is a list of the Item Exporters bundled with Scrapy. Some of them contain output examples, which assume you're exporting these two items: +.. skip: next .. code-block:: python Item(name="Color TV", price="1200") diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index e9423d64d..bc26bbebe 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -142,6 +142,7 @@ MongoDB collection is named after item class. The main point of this example is to show how to use :meth:`from_crawler` method and how to clean up the resources properly. +.. skip: next .. code-block:: python import pymongo diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst index 6bdc4bb9d..1201c926d 100644 --- a/docs/topics/link-extractors.rst +++ b/docs/topics/link-extractors.rst @@ -139,7 +139,7 @@ LxmlLinkExtractor .. code-block:: python def process_value(value): - m = re.search("javascript:goToPage\('(.*?)'", value) + m = re.search(r"javascript:goToPage\('(.*?)'", value) if m: return m.group(1) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index d7510ce0f..da0587aa4 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -606,6 +606,10 @@ See here the methods that you can override in your custom Files Pipeline: Here's a typical value of the ``results`` argument: + .. invisible-code-block: python + + from twisted.python.failure import Failure + .. code-block:: python [ diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 69e2d4f5e..f64da22d8 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -7,6 +7,8 @@ Common Practices This section documents common practices when using Scrapy. These are things that cover many topics and don't often fall into any other specific section. +.. skip: start + .. _run-from-script: Run Scrapy from a script @@ -231,6 +233,8 @@ different for different settings: .. seealso:: :ref:`run-from-script`. +.. skip: end + .. _distributed-crawls: Distributed crawls diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f05e27c73..99c7915df 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -76,6 +76,10 @@ Request objects :param cookies: the request cookies. These can be sent in two forms. + .. invisible-code-block: python + + from scrapy.http import Request + 1. Using a dict: .. code-block:: python @@ -917,6 +921,7 @@ If you want to simulate a HTML Form POST in your spider and send a couple of key-value fields, you can return a :class:`FormRequest` object (from your spider) like this: +.. skip: next .. code-block:: python return [ @@ -1000,6 +1005,7 @@ JsonRequest usage example Sending a JSON POST request with a JSON payload: +.. skip: next .. code-block:: python data = { diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d636dc301..420e85d37 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -71,6 +71,9 @@ do so by setting their :attr:`~scrapy.Spider.custom_settings` attribute: .. code-block:: python + import scrapy + + class MySpider(scrapy.Spider): name = "myspider" @@ -119,6 +122,7 @@ class or a function, there are two different ways you can specify that object: For example: +.. skip: next .. code-block:: python from mybot.pipelines.validate import ValidateMyItem @@ -417,6 +421,7 @@ a :class:`~scrapy.Request` based on its depth. The priority of a request is adjusted as follows: +.. skip: next .. code-block:: python request.priority = request.priority - (depth * DEPTH_PRIORITY) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 906aecdee..3400a205a 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -59,6 +59,9 @@ Let's take an example using :ref:`coroutines `: .. code-block:: python + import scrapy + + class SignalSpider(scrapy.Spider): name = "signals" start_urls = ["https://quotes.toscrape.com/page/1/"] diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 94bb4d44d..3f16efea5 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -269,6 +269,9 @@ this: .. code-block:: python + from scrapy.spiders import CrawlSpider + + class MySpider(CrawlSpider): handle_httpstatus_list = [404] diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 91f783b45..788bd7678 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -161,6 +161,9 @@ scrapy.Spider .. code-block:: python + import scrapy + + class MySpider(scrapy.Spider): name = "myspider" @@ -251,6 +254,7 @@ Return multiple Requests and items from a single callback: Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; to give data more structure you can use :class:`~scrapy.Item` objects: +.. skip: next .. code-block:: python import scrapy @@ -323,6 +327,7 @@ specify spider arguments when calling :class:`CrawlerProcess.crawl ` or :class:`CrawlerRunner.crawl `: +.. skip: next .. code-block:: python process = CrawlerProcess() @@ -478,9 +483,9 @@ Let's now take a look at an example CrawlSpider with rules: rules = ( # Extract links matching 'category.php' (but not matching 'subsection.php') # and follow links from them (since no callback means follow=True by default). - Rule(LinkExtractor(allow=("category\.php",), deny=("subsection\.php",))), + Rule(LinkExtractor(allow=(r"category\.php",), deny=(r"subsection\.php",))), # Extract links matching 'item.php' and parse them with the spider's method parse_item - Rule(LinkExtractor(allow=("item\.php",)), callback="parse_item"), + Rule(LinkExtractor(allow=(r"item\.php",)), callback="parse_item"), ) def parse_item(self, response): @@ -603,6 +608,7 @@ XMLFeedSpider example These spiders are pretty easy to use, let's have a look at one example: +.. skip: next .. code-block:: python from scrapy.spiders import XMLFeedSpider @@ -667,6 +673,7 @@ CSVFeedSpider example Let's see an example similar to the previous one, but using a :class:`CSVFeedSpider`: +.. skip: next .. code-block:: python from scrapy.spiders import CSVFeedSpider