diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5fc1285f..729682392 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,3 +16,9 @@ repos: rev: 5.12.0 hooks: - id: isort +- repo: https://github.com/adamchainz/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + additional_dependencies: + - black==22.12.0 diff --git a/docs/faq.rst b/docs/faq.rst index 8a9ba809b..836420567 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -35,8 +35,9 @@ for parsing HTML responses in Scrapy callbacks. You just have to feed the response's body into a ``BeautifulSoup`` object and extract whatever data you need from it. -Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser:: +Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parser: +.. code-block:: python from bs4 import BeautifulSoup import scrapy @@ -45,17 +46,12 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars class ExampleSpider(scrapy.Spider): name = "example" allowed_domains = ["example.com"] - start_urls = ( - 'http://www.example.com/', - ) + start_urls = ("http://www.example.com/",) def parse(self, response): # use lxml to get decent HTML parsing speed - soup = BeautifulSoup(response.text, 'lxml') - yield { - "url": response.url, - "title": soup.h1.string - } + soup = BeautifulSoup(response.text, "lxml") + yield {"url": response.url, "title": soup.h1.string} .. note:: @@ -109,11 +105,13 @@ basically means that it crawls in `DFO order`_. This order is more convenient in most cases. If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings:: +setting the following settings: + +.. code-block:: python DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleFifoDiskQueue' - SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.FifoMemoryQueue' + SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" + SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" While pending requests are below the configured values of :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or @@ -159,11 +157,13 @@ See also other suggestions at `StackOverflow`_. .. note:: Remember to disable :class:`scrapy.spidermiddlewares.offsite.OffsiteMiddleware` when you enable - your custom implementation:: + your custom implementation: + + .. code-block:: python SPIDER_MIDDLEWARES = { - 'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None, - 'myproject.middlewares.CustomOffsiteMiddleware': 500, + "scrapy.spidermiddlewares.offsite.OffsiteMiddleware": None, + "myproject.middlewares.CustomOffsiteMiddleware": 500, } .. _meet the installation requirements: https://github.com/andreasvc/pyre2#installation @@ -235,11 +235,13 @@ What does the response status code 999 means? 999 is a custom response status code used by Yahoo sites to throttle requests. Try slowing down the crawling speed by using a download delay of ``2`` (or -higher) in your spider:: +higher) in your spider: + +.. code-block:: python class MySpider(CrawlSpider): - name = 'myspider' + name = "myspider" download_delay = 2 @@ -351,19 +353,21 @@ How to split an item into multiple items in an item pipeline? input item. :ref:`Create a spider middleware ` instead, and use its :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` -method for this purpose. For example:: +method for this purpose. For example: + +.. code-block:: python from copy import deepcopy from itemadapter import is_item, ItemAdapter - class MultiplyItemsMiddleware: + class MultiplyItemsMiddleware: def process_spider_output(self, response, result, spider): for item in result: if is_item(item): adapter = ItemAdapter(item) - for _ in range(adapter['multiply_by']): + for _ in range(adapter["multiply_by"]): yield deepcopy(item) Does Scrapy support IPv6 addresses? diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index cfa6bfa83..495aad091 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -20,22 +20,24 @@ In order to show you what Scrapy brings to the table, we'll walk you through an example of a Scrapy Spider using the simplest way to run a spider. Here's the code for a spider that scrapes famous quotes from website -https://quotes.toscrape.com, following the pagination:: +https://quotes.toscrape.com, following the pagination + +.. code-block:: python import scrapy class QuotesSpider(scrapy.Spider): - name = 'quotes' + name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/tag/humor/', + "https://quotes.toscrape.com/tag/humor/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'author': quote.xpath('span/small/text()').get(), - 'text': quote.css('span.text::text').get(), + "author": quote.xpath("span/small/text()").get(), + "text": quote.css("span.text::text").get(), } next_page = response.css('li.next a::attr("href")').get() diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 901a170b4..f5e9b372e 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -177,7 +177,9 @@ that generates :class:`scrapy.Request ` objects from URLs, you can just define a :attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This list will then be used by the default implementation of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider:: +for your spider. + +.. code-block:: python from pathlib import Path @@ -187,13 +189,13 @@ for your spider:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): page = response.url.split("/")[-2] - filename = f'quotes-{page}.html' + filename = f"quotes-{page}.html" Path(filename).write_bytes(response.body) The :meth:`~scrapy.Spider.parse` method will be called to handle each @@ -438,7 +440,9 @@ extraction logic above into our spider. A Scrapy spider typically generates many dictionaries containing the data extracted from the page. To do that, we use the ``yield`` Python keyword -in the callback, as you can see below:: +in the callback, as you can see below: + +.. code-block:: python import scrapy @@ -446,16 +450,16 @@ in the callback, as you can see below:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', - 'https://quotes.toscrape.com/page/2/', + "https://quotes.toscrape.com/page/1/", + "https://quotes.toscrape.com/page/2/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } If you run this spider, it will output the extracted data with the log:: @@ -543,7 +547,9 @@ There is also an ``attrib`` property available '/page/2/' Let's see now our spider modified to recursively follow the link to the next -page, extracting data from it:: +page, extracting data from it: + +.. code-block:: python import scrapy @@ -551,18 +557,18 @@ page, extracting data from it:: class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -594,7 +600,9 @@ A shortcut for creating Requests -------------------------------- As a shortcut for creating Request objects you can use -:meth:`response.follow `:: +:meth:`response.follow ` + +.. code-block:: python import scrapy @@ -602,18 +610,18 @@ As a shortcut for creating Request objects you can use class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ - 'https://quotes.toscrape.com/page/1/', + "https://quotes.toscrape.com/page/1/", ] def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('span small::text').get(), - 'tags': quote.css('div.tags a.tag::text').getall(), + "text": quote.css("span.text::text").get(), + "author": quote.css("span small::text").get(), + "tags": quote.css("div.tags a.tag::text").getall(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, callback=self.parse) @@ -622,57 +630,67 @@ need to call urljoin. Note that ``response.follow`` just returns a Request instance; you still have to yield this Request. You can also pass a selector to ``response.follow`` instead of a string; -this selector should extract necessary attributes:: +this selector should extract necessary attributes: - for href in response.css('ul.pager a::attr(href)'): +.. code-block:: python + + for href in response.css("ul.pager a::attr(href)"): yield response.follow(href, callback=self.parse) For ```` elements there is a shortcut: ``response.follow`` uses their href -attribute automatically. So the code can be shortened further:: +attribute automatically. So the code can be shortened further: - for a in response.css('ul.pager a'): +.. code-block:: python + + for a in response.css("ul.pager a"): yield response.follow(a, callback=self.parse) To create multiple requests from an iterable, you can use -:meth:`response.follow_all ` instead:: +:meth:`response.follow_all ` instead: - anchors = response.css('ul.pager a') +.. code-block:: python + + anchors = response.css("ul.pager a") yield from response.follow_all(anchors, callback=self.parse) -or, shortening it further:: +or, shortening it further: - yield from response.follow_all(css='ul.pager a', callback=self.parse) +.. code-block:: python + + yield from response.follow_all(css="ul.pager a", callback=self.parse) More examples and patterns -------------------------- Here is another spider that illustrates callbacks and following links, -this time for scraping author information:: +this time for scraping author information: + +.. code-block:: python import scrapy class AuthorSpider(scrapy.Spider): - name = 'author' + name = "author" - start_urls = ['https://quotes.toscrape.com/'] + start_urls = ["https://quotes.toscrape.com/"] def parse(self, response): - author_page_links = response.css('.author + a') + author_page_links = response.css(".author + a") yield from response.follow_all(author_page_links, self.parse_author) - pagination_links = response.css('li.next a') + pagination_links = response.css("li.next a") yield from response.follow_all(pagination_links, self.parse) def parse_author(self, response): def extract_with_css(query): - return response.css(query).get(default='').strip() + return response.css(query).get(default="").strip() yield { - 'name': extract_with_css('h3.author-title::text'), - 'birthdate': extract_with_css('.author-born-date::text'), - 'bio': extract_with_css('.author-description::text'), + "name": extract_with_css("h3.author-title::text"), + "birthdate": extract_with_css(".author-born-date::text"), + "bio": extract_with_css(".author-description::text"), } This spider will start from the main page, it will follow all the links to the @@ -720,7 +738,9 @@ spider attributes by default. In this example, the value provided for the ``tag`` argument will be available via ``self.tag``. You can use this to make your spider fetch only quotes -with a specific tag, building the URL based on the argument:: +with a specific tag, building the URL based on the argument: + +.. code-block:: python import scrapy @@ -729,20 +749,20 @@ with a specific tag, building the URL based on the argument:: name = "quotes" def start_requests(self): - url = 'https://quotes.toscrape.com/' - tag = getattr(self, 'tag', None) + url = "https://quotes.toscrape.com/" + tag = getattr(self, "tag", None) if tag is not None: - url = url + 'tag/' + tag + url = url + "tag/" + tag yield scrapy.Request(url, self.parse) def parse(self, response): - for quote in response.css('div.quote'): + for quote in response.css("div.quote"): yield { - 'text': quote.css('span.text::text').get(), - 'author': quote.css('small.author::text').get(), + "text": quote.css("span.text::text").get(), + "author": quote.css("small.author::text").get(), } - next_page = response.css('li.next a::attr(href)').get() + next_page = response.css("li.next a::attr(href)").get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index dbee7146d..7713b1af1 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -106,12 +106,14 @@ Enforcing asyncio as a requirement If you are writing a :ref:`component ` that requires asyncio to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to :ref:`enforce it as a requirement `. For -example:: +example: + +.. code-block:: python from scrapy.utils.reactor import is_asyncio_reactor_installed - class MyComponent: + class MyComponent: def __init__(self): if not is_asyncio_reactor_installed(): raise ValueError( diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 0927ac2d2..8be89feb2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,9 +48,11 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ It works best during single-domain crawl. It does not work well with crawling many different domains in parallel -To apply the recommended priority queue use:: +To apply the recommended priority queue use: - SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' +.. code-block:: python + + SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" .. _broad-crawls-concurrency: @@ -71,7 +73,9 @@ many different domains in parallel, so you will want to increase it. How much to increase it will depend on how much CPU and memory your crawler will have available. -A good starting point is ``100``:: +A good starting point is ``100``: + +.. code-block:: python CONCURRENT_REQUESTS = 100 @@ -92,7 +96,9 @@ hitting DNS resolver timeouts. Possible solution to increase the number of threads handling DNS queries. The DNS queue will be processed faster speeding up establishing of connection and crawling overall. -To increase maximum thread pool size use:: +To increase maximum thread pool size use: + +.. code-block:: python REACTOR_THREADPOOL_MAXSIZE = 20 @@ -114,9 +120,11 @@ should not use ``DEBUG`` log level when preforming large broad crawls in production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. -To set the log level use:: +To set the log level use: - LOG_LEVEL = 'INFO' +.. code-block:: python + + LOG_LEVEL = "INFO" Disable cookies =============== @@ -126,7 +134,9 @@ doing broad crawls (search engine crawlers ignore them), and they improve performance by saving some CPU cycles and reducing the memory footprint of your Scrapy crawler. -To disable cookies use:: +To disable cookies use: + +.. code-block:: python COOKIES_ENABLED = False @@ -138,7 +148,9 @@ when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. -To disable retries use:: +To disable retries use: + +.. code-block:: python RETRY_ENABLED = False @@ -149,7 +161,9 @@ Unless you are crawling from a very slow connection (which shouldn't be the case for broad crawls) reduce the download timeout so that stuck requests are discarded quickly and free up capacity to process the next ones. -To reduce the download timeout use:: +To reduce the download timeout use: + +.. code-block:: python DOWNLOAD_TIMEOUT = 15 @@ -162,7 +176,9 @@ revisiting the site at a later crawl. This also help to keep the number of request constant per crawl batch, otherwise redirect loops may cause the crawler to dedicate too many resources on any specific domain. -To disable redirects use:: +To disable redirects use: + +.. code-block:: python REDIRECT_ENABLED = False @@ -179,7 +195,9 @@ Pages can indicate it in two ways: "main", "index" website pages. Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `:: +:ref:`AjaxCrawlMiddleware `: + +.. code-block:: python AJAXCRAWL_ENABLED = True diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 362190116..54fd5d663 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -617,7 +617,7 @@ Example: .. code-block:: python - COMMANDS_MODULE = 'mybot.commands' + COMMANDS_MODULE = "mybot.commands" .. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html @@ -636,10 +636,11 @@ The following example adds ``my_command`` command: from setuptools import setup, find_packages - setup(name='scrapy-mymodule', - entry_points={ - 'scrapy.commands': [ - 'my_command=my_scrapy_module.commands:MyCommand', - ], - }, - ) + setup( + name="scrapy-mymodule", + entry_points={ + "scrapy.commands": [ + "my_command=my_scrapy_module.commands:MyCommand", + ], + }, + ) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index ca301b827..1ed55f000 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -66,16 +66,18 @@ version mismatch, while :exc:`ValueError` may be better if the issue is the value of a setting. If your requirement is a minimum Scrapy version, you may use -:attr:`scrapy.__version__` to enforce your requirement. For example:: +:attr:`scrapy.__version__` to enforce your requirement. For example: + +.. code-block:: python from pkg_resources import parse_version import scrapy - class MyComponent: + class MyComponent: def __init__(self): - if parse_version(scrapy.__version__) < parse_version('2.7'): + if parse_version(scrapy.__version__) < parse_version("2.7"): raise RuntimeError( f"{MyComponent.__qualname__} requires Scrapy 2.7 or " f"later, which allow defining the process_spider_output " diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index c29a3a410..211a0f5f2 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -11,10 +11,13 @@ integrated way of testing your spiders by the means of contracts. This allows you to test each callback of your spider by hardcoding a sample url and check various constraints for how the callback processes the response. Each contract is prefixed with an ``@`` and included in the docstring. See the -following example:: +following example: + +.. code-block:: python def parse(self, response): - """ This function parses a sample response. Some contracts are mingled + """ + This function parses a sample response. Some contracts are mingled with this docstring. @url http://www.amazon.com/s?field-keywords=selfish+gene @@ -64,11 +67,13 @@ Custom Contracts If you find you need more power than the built-in Scrapy contracts you can create and load your own contracts in the project by using the -:setting:`SPIDER_CONTRACTS` setting:: +:setting:`SPIDER_CONTRACTS` setting: + +.. code-block:: python SPIDER_CONTRACTS = { - 'myproject.contracts.ResponseCheck': 10, - 'myproject.contracts.ItemValidate': 10, + "myproject.contracts.ResponseCheck": 10, + "myproject.contracts.ItemValidate": 10, } Each contract must inherit from :class:`~scrapy.contracts.Contract` and can @@ -111,22 +116,26 @@ Raise :class:`~scrapy.exceptions.ContractFail` from .. autoclass:: scrapy.exceptions.ContractFail Here is a demo contract which checks the presence of a custom header in the -response received:: +response received: + +.. code-block:: python from scrapy.contracts import Contract from scrapy.exceptions import ContractFail + class HasHeaderContract(Contract): - """ Demo contract which checks the presence of a custom header - @has_header X-CustomHeader + """ + Demo contract which checks the presence of a custom header + @has_header X-CustomHeader """ - name = 'has_header' + name = "has_header" def pre_process(self, response): for header in self.args: if header not in response.headers: - raise ContractFail('X-CustomHeader not present') + raise ContractFail("X-CustomHeader not present") .. _detecting-contract-check-runs: @@ -135,14 +144,17 @@ Detecting check runs When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is set to the ``true`` string. You can use :data:`os.environ` to perform any change to -your spiders or your settings when ``scrapy check`` is used:: +your spiders or your settings when ``scrapy check`` is used: + +.. code-block:: python import os import scrapy + class ExampleSpider(scrapy.Spider): - name = 'example' + name = "example" def __init__(self): - if os.environ.get('SCRAPY_CHECK'): + if os.environ.get("SCRAPY_CHECK"): pass # Do some scraper adjustments when a check is running diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a1ba4ba5c..a0c005204 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -58,49 +58,58 @@ There are several use cases for coroutines in Scrapy. Code that would return Deferreds when written for previous Scrapy versions, such as downloader middlewares and signal handlers, can be rewritten to be -shorter and cleaner:: +shorter and cleaner: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: def _update_item(self, data, item): adapter = ItemAdapter(item) - adapter['field'] = data + adapter["field"] = data return item def process_item(self, item, spider): adapter = ItemAdapter(item) - dfd = db.get_some_data(adapter['id']) + dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) return dfd -becomes:: +becomes: + +.. code-block:: python from itemadapter import ItemAdapter + class DbPipeline: async def process_item(self, item, spider): adapter = ItemAdapter(item) - adapter['field'] = await db.get_some_data(adapter['id']) + adapter["field"] = await db.get_some_data(adapter["id"]) return item Coroutines may be used to call asynchronous code. This includes other coroutines, functions that return Deferreds and functions that return :term:`awaitable objects ` such as :class:`~asyncio.Future`. -This means you can use many useful Python libraries providing such code:: +This means you can use many useful Python libraries providing such code: + +.. code-block:: python class MySpiderDeferred(Spider): # ... async def parse(self, response): - additional_response = await treq.get('https://additional.url') + additional_response = await treq.get("https://additional.url") additional_data = await treq.content(additional_response) # ... use response and additional_data to yield items and requests + class MySpiderAsyncio(Spider): # ... async def parse(self, response): async with aiohttp.ClientSession() as session: - async with session.get('https://additional.url') as additional_response: + async with session.get("https://additional.url") as additional_response: additional_data = await additional_response.text() # ... use response and additional_data to yield items and requests @@ -192,7 +201,9 @@ while maintaining support for older Scrapy versions, you may define :term:`asynchronous generator` version of that method with an alternative name: ``process_spider_output_async``. -For example:: +For example: + +.. code-block:: python class UniversalSpiderMiddleware: def process_spider_output(self, response, result, spider): diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index edbcaf432..b133fcc1e 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -5,21 +5,24 @@ Debugging Spiders ================= This document explains the most common techniques for debugging spiders. -Consider the following Scrapy spider below:: +Consider the following Scrapy spider below: + +.. code-block:: python import scrapy from myproject.items import MyItem + class MySpider(scrapy.Spider): - name = 'myspider' + name = "myspider" start_urls = ( - 'http://example.com/page1', - 'http://example.com/page2', - ) + "http://example.com/page1", + "http://example.com/page2", + ) def parse(self, response): # - # collect `item_urls` + # collect `item_urls` for item_url in item_urls: yield scrapy.Request(item_url, self.parse_item) @@ -28,7 +31,9 @@ Consider the following Scrapy spider below:: item = MyItem() # populate `item` fields # and extract item_details_url - yield scrapy.Request(item_details_url, self.parse_details, cb_kwargs={'item': item}) + yield scrapy.Request( + item_details_url, self.parse_details, cb_kwargs={"item": item} + ) def parse_details(self, response, item): # populate more `item` fields @@ -103,10 +108,13 @@ showing the response received and the output. How to debug the situation when .. highlight:: python Fortunately, the :command:`shell` is your bread and butter in this case (see -:ref:`topics-shell-inspect-response`):: +:ref:`topics-shell-inspect-response`): + +.. code-block:: python from scrapy.shell import inspect_response + def parse_details(self, response, item=None): if item: # populate more `item` fields @@ -121,10 +129,13 @@ Open in browser Sometimes you just want to see how a certain response looks in a browser, you can use the ``open_in_browser`` function for that. Here is an example of how -you would use it:: +you would use it: + +.. code-block:: python from scrapy.utils.response import open_in_browser + def parse_details(self, response): if "item name" not in response.body: open_in_browser(response) @@ -138,14 +149,16 @@ Logging Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be -available in all future runs should they be necessary again:: +available in all future runs should they be necessary again: + +.. code-block:: python def parse_details(self, response, item=None): if item: # populate more `item` fields return item else: - self.logger.warning('No item received for %s', response.url) + self.logger.warning("No item received for %s", response.url) For more information, check the :ref:`topics-logging` section. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 9bf97c628..39e7b7d3c 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -237,17 +237,19 @@ on the request and open ``Open in new tab`` to get a better overview. :alt: JSON-object returned from the quotes.toscrape API With this response we can now easily parse the JSON-object and -also request each page to get every quote on the site:: +also request each page to get every quote on the site: + +.. code-block:: python import scrapy import json class QuoteSpider(scrapy.Spider): - name = 'quote' - allowed_domains = ['quotes.toscrape.com'] + name = "quote" + allowed_domains = ["quotes.toscrape.com"] page = 1 - start_urls = ['https://quotes.toscrape.com/api/quotes?page=1'] + start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): data = json.loads(response.text) @@ -275,7 +277,9 @@ requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the :meth:`~scrapy.Request.from_curl()` method to generate an equivalent -request:: +request: + +.. code-block:: python from scrapy import Request @@ -286,7 +290,8 @@ request:: "-Requested-With: XMLHttpRequest' -H 'Proxy-Authorization: Basic QFRLLTAzM" "zEwZTAxLTk5MWUtNDFiNC1iZWRmLTJjNGI4M2ZiNDBmNDpAVEstMDMzMTBlMDEtOTkxZS00MW" "I0LWJlZGYtMmM0YjgzZmI0MGY0' -H 'Connection: keep-alive' -H 'Referer: http" - "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'") + "://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'" + ) Alternatively, if you want to know the arguments needed to recreate that request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs` diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 986da0476..e1c481c37 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -17,10 +17,12 @@ To activate a downloader middleware component, add it to the :setting:`DOWNLOADER_MIDDLEWARES` setting, which is a dict whose keys are the middleware class paths and their values are the middleware orders. -Here's an example:: +Here's an example: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, + "myproject.middlewares.CustomDownloaderMiddleware": 543, } The :setting:`DOWNLOADER_MIDDLEWARES` setting is merged with the @@ -42,11 +44,13 @@ previous (or subsequent) middleware being applied. If you want to disable a built-in middleware (the ones defined in :setting:`DOWNLOADER_MIDDLEWARES_BASE` and enabled by default) you must define it in your project's :setting:`DOWNLOADER_MIDDLEWARES` setting and assign ``None`` -as its value. For example, if you want to disable the user-agent middleware:: +as its value. For example, if you want to disable the user-agent middleware: + +.. code-block:: python DOWNLOADER_MIDDLEWARES = { - 'myproject.middlewares.CustomDownloaderMiddleware': 543, - 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, + "myproject.middlewares.CustomDownloaderMiddleware": 543, + "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": None, } Finally, keep in mind that some middlewares may need to be enabled through a @@ -226,20 +230,25 @@ There is support for keeping multiple cookie sessions per spider by using the :reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar (session), but you can pass an identifier to use different ones. -For example:: +For example: + +.. code-block:: python for i, url in enumerate(urls): - yield scrapy.Request(url, meta={'cookiejar': i}, - callback=self.parse_page) + yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page) Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep -passing it along on subsequent requests. For example:: +passing it along on subsequent requests. For example: + +.. code-block:: python def parse_page(self, response): # do some processing - return scrapy.Request("http://www.example.com/otherpage", - meta={'cookiejar': response.meta['cookiejar']}, - callback=self.parse_other_page) + return scrapy.Request( + "http://www.example.com/otherpage", + meta={"cookiejar": response.meta["cookiejar"]}, + callback=self.parse_other_page, + ) .. setting:: COOKIES_ENABLED @@ -339,16 +348,19 @@ HttpAuthMiddleware domain of the first request, which will work for some spiders but not for others. In the future the middleware will produce an error instead. - Example:: + Example: + + .. code-block:: python from scrapy.spiders import CrawlSpider + class SomeIntranetSiteSpider(CrawlSpider): - http_user = 'someuser' - http_pass = 'somepass' - http_auth_domain = 'intranet.example.com' - name = 'intranet.example.com' + http_user = "someuser" + http_pass = "somepass" + http_auth_domain = "intranet.example.com" + name = "intranet.example.com" # .. rest of the spider code omitted ... @@ -792,7 +804,9 @@ If you want to handle some redirect status codes in your spider, you can specify these in the ``handle_httpstatus_list`` spider attribute. For example, if you want the redirect middleware to ignore 301 and 302 -responses (and pass them through to your spider) you can do this:: +responses (and pass them through to your spider) you can do this: + +.. code-block:: python class MySpider(CrawlSpider): handle_httpstatus_list = [301, 302] diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index ea5d06210..9be0ed058 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -119,16 +119,20 @@ data from it depends on the type of response: ` as usual. - If the response is JSON, use :func:`json.loads` to load the desired data from - :attr:`response.text `:: + :attr:`response.text `: + + .. code-block:: python data = json.loads(response.text) If the desired data is inside HTML or XML code embedded within JSON data, you can load that HTML or XML code into a :class:`~scrapy.Selector` and then - :ref:`use it ` as usual:: + :ref:`use it ` as usual: - selector = Selector(data['html']) + .. code-block:: python + + selector = Selector(data["html"]) - If the response is JavaScript, or HTML with a ``