From 3e05a2ecf69bc6ccc791575fed931519d14792f1 Mon Sep 17 00:00:00 2001 From: Alexandru Cepoi Date: Tue, 12 Jun 2012 18:28:10 +0200 Subject: [PATCH 1/3] update docs for parse command --- docs/topics/commands.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 929a03250..bac238fb8 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -368,15 +368,25 @@ Supported options: * ``--nolinks``: don't show scraped items +* ``--depth`` or ``-d``: depth level for which the requests should be followed + recursively (default: 1) + +* ``--verbose`` or ``-v``: display information for each depth level + Usage example:: $ scrapy parse http://www.example.com/ -c parse_item [ ... scrapy log lines crawling example.com spider ... ] - # Scraped Items - callback: parse ------------------------------------------------------------ - MyItem({'name': u"Example item", + + >>> STATUS DEPTH LEVEL 1 <<< + # Scraped Items ------------------------------------------------------------ + [{'name': u'Example item', 'category': u'Furniture', - 'length': u'12 cm'} - ) + 'length': u'12 cm'}] + + # Requests ----------------------------------------------------------------- + [] + .. command:: settings From f4faa19e31b3192332e87f9fe78ff0914a848bc6 Mon Sep 17 00:00:00 2001 From: Alexandru Cepoi Date: Thu, 21 Jun 2012 20:03:33 +0200 Subject: [PATCH 2/3] added docs topic debugging spiders --- docs/index.rst | 4 ++ docs/topics/debug.rst | 125 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 docs/topics/debug.rst diff --git a/docs/index.rst b/docs/index.rst index 80b5d513d..317ccb2e1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -128,6 +128,7 @@ Solving specific problems :hidden: faq + topics/debug topics/firefox topics/firebug topics/leaks @@ -139,6 +140,9 @@ Solving specific problems :doc:`faq` Get answers to most frequently asked questions. +:doc:`topics/debug` + Learn how to debug common problems of your scrapy spider. + :doc:`topics/firefox` Learn how to scrape with Firefox and some useful add-ons. diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst new file mode 100644 index 000000000..6d94093c8 --- /dev/null +++ b/docs/topics/debug.rst @@ -0,0 +1,125 @@ +.. _topics-debug: + +================= +Debugging Spiders +================= + +This document explains the most common techniques for debugging spiders. +Consider the following scrapy spider below:: + + class MySpider(BaseSpider): + name = 'myspider' + start_urls = ( + 'http://example.com/page1', + 'http://example.com/page2', + ) + + def parse(self, response): + # collect `item_urls` + for item_url in item_urls: + yield Request(url=item_url, callback=self.parse_item) + + def parse_item(self, response): + item = MyItem() + # populate `item` fields + yield Request(url=item_details_url, meta={'item': item}, + callback=self.parse_details) + + def parse_details(self, response): + item = response.meta['item'] + # populate more `item` fields + return item + +Basically this is a simple spider which parses two pages of items (the +start_urls). Items also have a details page with additional information, so we +use the ``meta`` functionality of :class:`~scrapy.http.Request` to pass a +partially populated item. + + +Parse Command +============= + +The most basic way of checking the output of your spider is to use the +:command:`parse` command. It allows to check the behaviour of different parts +of the spider at the method level. It has the advantage of being flexible and +simple to use, but does not allow debugging code inside a method. + +In order to see the item scraped from a specific url:: + + $ scrapy --spider=myspider -c parse_item -d 2 + [ ... scrapy log lines crawling example.com spider ... ] + + >>> STATUS DEPTH LEVEL 2 <<< + # Scraped Items ------------------------------------------------------------ + [{'url': }] + + # Requests ----------------------------------------------------------------- + [] + +Using the ``--verbose`` or ``-v`` option we can see the status at each depth level:: + + $ scrapy --spider=myspider -c parse_item -d 2 -v + [ ... scrapy log lines crawling example.com spider ... ] + + >>> DEPTH LEVEL: 1 <<< + # Scraped Items ------------------------------------------------------------ + [] + + # Requests ----------------------------------------------------------------- + [] + + + >>> DEPTH LEVEL: 2 <<< + # Scraped Items ------------------------------------------------------------ + [{'url': }] + + # Requests ----------------------------------------------------------------- + [] + +Checking items scraped from a single start_url, can also be easily achieved +using:: + + $ scrapy --spider=myspider -d 3 'http://example.com/page1' + + +Scrapy Shell +============ + +While the :command:`parse` command is very useful for checking behaviour of a +spider, it is of little help to check what happens inside a callback, besides +showing the reponse received and the output. How to debug the situation when +``parse_details`` sometimes receives no item? + +Fortunately, the :command:`shell` is your bread and butter in this case (see +:ref:`topics-shell-inspect-response`):: + + from scrapy.shell import inspect_response + + def parse_details(self, response): + item = response.meta.get('item', None) + if item: + # populate more `item` fields + return item + else: + inspect_response(response, self) + + +Logging +======= + +Logging is another useful option for getting information about your spider run. +Although not as convenient, it comes with the advantage that the logs will be +available in all future runs should they be necessary again:: + + from scrapy import log + + def parse_details(self, response): + item = response.meta.get('item', None) + if item: + # populate more `item` fields + return item + else: + self.log('No item received for %s' % response.url, + level=log.WARNING) + +For more information, check the :ref:`topics-logging` section. From 2e05cf56852cd53bc91a9b9e1c1024fb7a1dfbeb Mon Sep 17 00:00:00 2001 From: Alexandru Cepoi Date: Thu, 21 Jun 2012 20:06:50 +0200 Subject: [PATCH 3/3] fix small bug with parse command --- scrapy/commands/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 28bc1467e..3e01c771d 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -154,7 +154,7 @@ class Command(ScrapyCommand): cb = cb if callable(cb) else getattr(self.spider, cb, None) if not cb: - log.msg('Cannot find callback %r in spider: %s' % (callback, spider.name)) + log.msg('Cannot find callback %r in spider: %s' % (callback, self.spider.name)) # parse items and requests depth = response.meta['_depth']