From a2e64525544c8fa5ba0d9ad85aedac0801b7e3de Mon Sep 17 00:00:00 2001 From: Darshan Chaudhary Date: Thu, 29 Oct 2015 15:40:07 +0530 Subject: [PATCH] Include signal example --- docs/topics/media-pipeline.rst | 2 +- docs/topics/signals.rst | 31 +++++++++++++++++++++++++++++++ scrapy/utils/misc.py | 2 +- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 4ee4f1758..5ed6ce97d 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -7,7 +7,7 @@ Downloading and processing files and images .. currentmodule:: scrapy.pipelines.images Scrapy provides reusable :doc:`item pipelines ` for -downloading fies attached to a particular item (for example, when you scrape +downloading files attached to a particular item (for example, when you scrape products and also want to download their images locally). These pipelines share a bit of functionality and structure (we refer to them as media pipelines), but typically you'll either use the Files Pipeline or the Images Pipeline. diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 5dd3b9ef5..19d5e8df9 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -16,6 +16,37 @@ deliver the arguments that the handler receives. You can connect to signals (or send your own) through the :ref:`topics-api-signals`. +Here is a simple example showing how you can catch signals and perform some action: +:: + + from scrapy import signals + from scrapy import Spider + + + class DmozSpider(Spider): + name = "dmoz" + allowed_domains = ["dmoz.org"] + start_urls = [ + "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", + "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/", + ] + + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs) + crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed) + return spider + + + def spider_closed(self, spider): + spider.logger.info('Spider closed: %s', spider.name) + + + def parse(self, response): + pass + + Deferred signal handlers ======================== diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 303a413d8..f20070b5d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,4 +1,4 @@ -"""Helper functions which doesn't fit anywhere else""" +"""Helper functions which don't fit anywhere else""" import re import hashlib from importlib import import_module