diff --git a/scrapy/trunk/docs/_ext/scrapydocs.py b/scrapy/trunk/docs/_ext/scrapydocs.py index b54952422..8358a01af 100644 --- a/scrapy/trunk/docs/_ext/scrapydocs.py +++ b/scrapy/trunk/docs/_ext/scrapydocs.py @@ -4,3 +4,18 @@ def setup(app): rolename = "setting", indextemplate = "pair: %s; setting", ) + app.add_crossref_type( + directivename = "topic", + rolename = "topic", + indextemplate = "pair: %s; topic", + ) + app.add_crossref_type( + directivename = "signal", + rolename = "signal", + indextemplate = "pair: %s; signal", + ) + app.add_crossref_type( + directivename = "exception", + rolename = "exception", + indextemplate = "pair: %s; exception", + ) diff --git a/scrapy/trunk/docs/conf.py b/scrapy/trunk/docs/conf.py index 4e2cb2d28..c2e155a17 100644 --- a/scrapy/trunk/docs/conf.py +++ b/scrapy/trunk/docs/conf.py @@ -46,9 +46,9 @@ copyright = u'2008, Insophia' # built documents. # # The short X.Y version. -version = '1.0' +version = '0.7' # The full version, including alpha/beta/rc tags. -release = '1.0' +release = '0.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/scrapy/trunk/docs/ref/exceptions.rst b/scrapy/trunk/docs/ref/exceptions.rst new file mode 100644 index 000000000..31ffee63c --- /dev/null +++ b/scrapy/trunk/docs/ref/exceptions.rst @@ -0,0 +1,70 @@ +.. _exceptions: + +Available Exceptions +==================== + +Here's a list of all exceptions included in Scrapy and their usage. + +.. exception:: DontCloseDomain + +DontCloseDomain +--------------- + +This exception can be raised by any handler of the :signal:`domain_idle` signal +to avoid the domain from being closed at this time, and wait for the next idle +state. + +.. exception:: DropItem + +DropItem +-------- + +The exception that must be raised by item pipeline stages to stop processing an +Item. For more information see :topic:`itempipeline`. + +.. exception:: NotConfigured + +NotConfigured +------------- + +This exception can be raised by some components to indicate that they will +remain disabled. Those component include: + + * Extensions + * Item pipelines + * Downloader middlwares + * Spider middlewares + +The exception must be raised in the component constructor. + +.. exception:: HttpException + +HttpException +------------- + +This exception is raised by the downloader when a non-200 response has been +downloaded. + +.. exception:: IgnoreRequest + +IgnoreRequest +------------- + +This exception can be raised by the Scheduler or any downlaoder middleware to +indicate that the request should be ignored. + +.. exception:: NotSupported + +NotSupported +------------ + +This exception is raised to indicate an unsupported feature. + +.. exception:: UsageError + +UsageError +---------- + +This exception indicates a incorrect usage of the Scrapy API. + + diff --git a/scrapy/trunk/docs/ref/index.rst b/scrapy/trunk/docs/ref/index.rst index 050ebe28b..e5c69e122 100644 --- a/scrapy/trunk/docs/ref/index.rst +++ b/scrapy/trunk/docs/ref/index.rst @@ -4,4 +4,6 @@ Reference .. toctree:: :maxdepth: 1 + exceptions settings + signals diff --git a/scrapy/trunk/docs/ref/settings.rst b/scrapy/trunk/docs/ref/settings.rst index 2f53d0a23..d3a0f091b 100644 --- a/scrapy/trunk/docs/ref/settings.rst +++ b/scrapy/trunk/docs/ref/settings.rst @@ -1,10 +1,10 @@ .. _settings: -Available settings +Available Settings ================== -Here's a full list of all available Scrapy settings, in alphabetical order, -along with their default values and the scope where they apply. +Here's a list of all available Scrapy settings, in alphabetical order, along +with their default values and the scope where they apply. The scope, where available, shows where the setting is being used, if it's tied to any particular component. In that case the module of that component will be diff --git a/scrapy/trunk/docs/ref/signals.rst b/scrapy/trunk/docs/ref/signals.rst new file mode 100644 index 000000000..56c44b9cc --- /dev/null +++ b/scrapy/trunk/docs/ref/signals.rst @@ -0,0 +1,175 @@ +.. _signals: + +Available Signals +================= + +Scrapy uses signals extensively to notify when certain actions occur. You can +catch some of those signals in your Scrapy project or extension to perform +additional tasks or extend Scrapy to add functionality not provided out of the +box. + +Here's a list of signals used in Scrapy and their meaning, in alphabetical +order. + +.. signal:: domain_closed + +domain_closed +------------- + +Arguments: + * ``domain`` - the domain (of the spider) which has been closed + * ``spider`` - the spider which has been closed + +Sent right after a spider/domain has been closed. + +.. signal:: domain_open + +domain_open +----------- + +Arguments: + * ``domain`` - the domain (of the spider) which is about to be opened + * ``spider`` - the spider which is about to be opened + +Sent right before a spider has been opened for crawling. + +.. signal:: domain_opened + +domain_opened +------------- + +Arguments: + * ``domain`` - the domain (of the spider) which has been opened + * ``spider`` - the spider which has been opened + +Sent right after a spider has been opened for crawling. + +.. signal:: domain_idle + +domain_idle +----------- + +Arguments: + * ``domain`` - the domain (of the spider) which has gone idle + * ``spider`` - the spider which has gone idle + +Sent when a domain has no further: + * requests waiting to be downloaded + * requests scheduled + * items being processed in the item pipeline + +If any handler of this signals raises a :exception:`DontCloseDomain` the domain +won't be closed at this time and will wait until another idle signal is sent. +Otherwise (if no handler raises :exception:`DontCloseDomain`) the domain will +be closed immediately after all handlers of ``domain_idle`` have finished, and +a :signal:`domain_closed` will thus be sent. + +.. signal:: engine_started + +engine_started +-------------- + +Arguments: ``None`` + +Sent when the Scrapy engine is started (for example, when a crawling +process has started). + +.. signal:: engine_stopped + +engine_stopped +-------------- + +Arguments: ``None`` + +Sent when the Scrapy engine is stopped (for example, when a crawling +process has started). + +.. signal:: request_received + +request_received +---------------- + +Arguments: + * ``request`` - the ``HTTPRequest`` received + * ``spider`` - the spider which generated the request + * ``response`` - the ``HTTPResponse`` fed to the spider which generated the + request + +Sent when the engine receives a ``HTTPRequest`` from a spider. + +.. signal:: request_uploaded + +request_uploaded +---------------- + +Arguments: + * ``request`` - the ``HTTPRequest`` uploaded/sent + * ``spider`` - the spider which generated the request + +Sent right after the download has sent a ``HTTPRequest``. + +.. signal:: response_received + +response_received +----------------- + +Arguments: + * ``response`` - the ``HTTPResponse`` received + * ``spider`` - the spider for which the response is intended + +Sent when the engine receives a new ``HTTPResponse`` from the downloader. + +.. signal:: response_downloaded + +response_downloaded +------------------- + +Arguments: + * ``response`` - the ``HTTPResponse`` downloaded + * ``spider`` - the spider for which the response is intended + +Sent by the downloader right after a ``HTTPResponse`` is downloaded. + +.. signal:: item_scraped + +item_scraped +------------ + +Arguments: + * ``item`` - the item scraped + * ``spider`` - the spider which scraped the item + * ``response`` - the response from which the item was scraped + +Sent when the engine receives a new scraped item from the spider, and right +before the item is sent to the :topic:`itempipeline`. + +.. signal:: item_passed + +item_passed +----------- + +Arguments: + * ``item`` - the item passed + * ``spider`` - the spider which scraped the item + * ``response`` - the response from which the item was scraped + * ``pipe_output`` - the output of the item pipeline. Typically, this points to + the same ``item`` object, unless some pipeline stage created a new item. + +Sent after an item has passed al the :topic:`itempipeline` stages without +being dropped. + +.. signal:: item_dropped + +item_dropped +------------ + +Arguments: + * ``item`` - the item dropped + * ``spider`` - the spider which scraped the item + * ``response`` - the response from which the item was scraped + * ``exception`` - the exception that caused the item to be dropped (which must inherit from :exception:`DropItem`) + +Sent after an item has dropped from the :topic:`itempipeline` when some stage +raised a :exception:`DropItem` exception. + + diff --git a/scrapy/trunk/docs/topics/index.rst b/scrapy/trunk/docs/topics/index.rst index 5aedaa06a..5e6aa58fd 100644 --- a/scrapy/trunk/docs/topics/index.rst +++ b/scrapy/trunk/docs/topics/index.rst @@ -4,6 +4,7 @@ Topics .. toctree:: :maxdepth: 1 + itempipeline selectors settings spiders diff --git a/scrapy/trunk/docs/topics/itempipeline.rst b/scrapy/trunk/docs/topics/itempipeline.rst new file mode 100644 index 000000000..92cc60da0 --- /dev/null +++ b/scrapy/trunk/docs/topics/itempipeline.rst @@ -0,0 +1,48 @@ +============= +Item Pipeline +============= + +After an item has been scraped by a spider it is sent to the Item Pipeline +which process it through several stages that are executed sequentially. + +Item pipeline are usually implemented on each project. Typical usage for item +pipelines are: + + * HTML cleansing + * validation + * persistence (storing the scraped item) + +To implement an item pipeline you need to implement a class which contains a +``process_item`` method. That method must either return a ScrapedItem (or any +descendant class) object or raise a :exception:`DropItem` exception. Dropped +items are no longer processed by further pipeline stages. + +Let's take a look at following hypotetic pipeline that adjusts the ``price`` +attribute for those items that do not include VAT (``price_excludes_vat`` +attribute), and drops those items which don't contain a price:: + + from scrapy.core.exceptions import DropItem + + class PricePipeline(object): + + vat_factor = 1.15 + + def process_item(self, domain, response, item): + if item.price: + if item.price_excludes_vat: + item.price = item.price * self.vat_factor + return item + else: + raise DropItem("Missing price in %s" % item) + + def open_domain(self, domain): + pass + + def close_domains(self, domain): + pass + + +You may also noticed the two other methods that a item pipeline class can +contain: ``open_domain`` which is called when the domain for that spider is +open, and ``close_domain`` which is called when it's closed. +