diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 390a5f541..0a83b248e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -364,6 +364,52 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`:: Spider arguments can also be passed through the Scrapyd ``schedule.json`` API. See `Scrapyd documentation`_. +scrapy-spider-metadata parameters +--------------------------------- + +Another alternative to pass spider arguments is the library `scrapy-spider-metadata`_. + +This allows for Scrapy spiders to define, validate, document and pre-process +their arguments as Pydantic models. + +The example shows how to define typed parameters where a string argument +is automatically converted to an integer: + +.. code-block:: python + + import scrapy + from pydantic import BaseModel + from scrapy_spider_metadata import Args + + + class MyParams(BaseModel): + pages: int + + + class BookSpider(Args[MyParams], scrapy.Spider): + name = "bookspider" + start_urls = ["http://books.toscrape.com/catalogue"] + + async def start(self): + for start_url in self.start_urls: + for index in range(1, self.args.pages + 1): + yield scrapy.Request(f"{start_url}/page-{index}.html") + + def parse(self, response): + book_links = response.css("article.product_pod h3 a::attr(href)").getall() + for book_link in book_links: + yield response.follow(book_link, self.parse_book) + + def parse_book(self, response): + yield { + "title": response.css("h1::text").get(), + "price": response.css("p.price_color::text").get(), + } + +This spider can be called from the command line:: + + scrapy crawl bookspider -a pages=2 + .. _start-requests: Start requests @@ -938,6 +984,7 @@ Combine SitemapSpider with other sources of urls: def parse_other(self, response): pass # ... scrape other here ... +.. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html .. _Sitemaps: https://www.sitemaps.org/index.html .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index .. _robots.txt: https://www.robotstxt.org/