From bb9a732edfc6f0f85f4e7356722a6ab09bf4c4c8 Mon Sep 17 00:00:00 2001 From: Ismael Carnales Date: Mon, 16 Feb 2009 11:51:07 +0000 Subject: [PATCH] updated tutorial with new googledir project from r853 --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40855 --- .../trunk/docs/intro/tutorial/tutorial1.rst | 20 ++--- .../trunk/docs/intro/tutorial/tutorial2.rst | 77 +++++++++--------- .../trunk/docs/intro/tutorial/tutorial3.rst | 50 +++++++----- .../trunk/docs/intro/tutorial/tutorial4.rst | 79 ++++++++++--------- 4 files changed, 121 insertions(+), 105 deletions(-) diff --git a/scrapy/trunk/docs/intro/tutorial/tutorial1.rst b/scrapy/trunk/docs/intro/tutorial/tutorial1.rst index cbf3b929a..b24a04ce7 100644 --- a/scrapy/trunk/docs/intro/tutorial/tutorial1.rst +++ b/scrapy/trunk/docs/intro/tutorial/tutorial1.rst @@ -15,14 +15,14 @@ We'll assume that Scrapy is already installed in your system, if not see For starting a new project, enter the directory where you'd like your project to be located, and run:: - $ scrapy-admin.py startproject google + $ scrapy-admin.py startproject googledir As long as Scrapy is well installed and the path is set, this will create a -``google`` directory with the following contents:: +``googledir`` directory with the following contents:: - google/ + googledir/ scrapy-ctl.py - google/ + googledir/ __init__.py items.py pipelines.py @@ -38,17 +38,17 @@ These are basically: different tasks (like "genspider", "crawl" and "parse"). We'll talk more about this later. -* ``google/``: the project's actual python module, you'll import your code from here. +* ``googledir/``: the project's actual python module, you'll import your code from here. -* ``google/items.py``: were you define the different kinds of items you're going to scrape. +* ``googledir/items.py``: were you define the different kinds of items you're going to scrape. -* ``google/pipelines.py``: were you define your item pipelines. +* ``googledir/pipelines.py``: were you define your item pipelines. -* ``google/settings.py``: the project's settings file. +* ``googledir/settings.py``: the project's settings file. -* ``google/spiders/``: directory where you'll later place your spiders. +* ``googledir/spiders/``: directory where you'll later place your spiders. -* ``google/templates/``: directory containing some templates for newly created +* ``googledir/templates/``: directory containing some templates for newly created spiders, and where you can put your own. diff --git a/scrapy/trunk/docs/intro/tutorial/tutorial2.rst b/scrapy/trunk/docs/intro/tutorial/tutorial2.rst index 30218b4e0..25b5e75f8 100644 --- a/scrapy/trunk/docs/intro/tutorial/tutorial2.rst +++ b/scrapy/trunk/docs/intro/tutorial/tutorial2.rst @@ -9,34 +9,34 @@ on your project's directory and run:: ./scrapy-ctl.py genspider google_directory google.com -This should create a file called google_directory.py under ``google/spiders`` +This should create a file called google_directory.py under ``googledir/spiders`` directory looking like this:: - # -*- coding: utf8 -*- - import re + # -*- coding: utf8 -*- + import re - from scrapy.xpath import HtmlXPathSelector - from google.items import GoogleItem - from scrapy.link.extractors import RegexLinkExtractor - from scrapy.contrib.spiders import CrawlSpider, Rule + from scrapy.xpath import HtmlXPathSelector + from scrapy.link.extractors import RegexLinkExtractor + from scrapy.contrib.spiders import CrawlSpider, Rule + from googledir.items import GoogledirItem - class GoogleDirectorySpider(CrawlSpider): - domain_name = 'google.com' - start_urls = ['http://www.google.com/'] + class GoogleDirectorySpider(CrawlSpider): + domain_name = 'google.com' + start_urls = ['http://www.google.com/'] - rules = ( - Rule(RegexLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True), - ) + rules = ( + Rule(RegexLinkExtractor(allow=(r'Items/', )), 'parse_item', follow=True), + ) - def parse_item(self, response): - xs = HtmlXPathSelector(response) - i = GoogleItem() - #i.attribute('site_id', xs.x('//input[@id="sid"]/@value')) - #i.attribute('name', xs.x('//div[@id="name"]')) - #i.attribute('description', xs.x('//div[@id="description"]')) - return [i] + def parse_item(self, response): + i = GoogledirItem() + #xs = HtmlXPathSelector(response) + #i.attribute('site_id', xs.x('//input[@id="sid"]/@value')) + #i.attribute('name', xs.x('//div[@id="name"]')) + #i.attribute('description', xs.x('//div[@id="description"]')) + return [i] - SPIDER = GoogleDirectorySpider() + SPIDER = GoogleDirectorySpider() Now, let's explain a bit what this is all about. @@ -124,28 +124,29 @@ decided to extract more links from them with follow=True. Until now, our spider would look something like:: - # -*- coding: utf8 -*- - from scrapy.xpath import HtmlXPathSelector - from google.items import GoogleItem - from scrapy.link.extractors import RegexLinkExtractor - from scrapy.contrib.spiders import CrawlSpider, Rule + # -*- coding: utf8 -*- + import re - class GoogleDirectorySpider(CrawlSpider): - domain_name = 'google.com' - start_urls = ['http://www.google.com/dirhp'] + from scrapy.xpath import HtmlXPathSelector + from scrapy.link.extractors import RegexLinkExtractor + from scrapy.contrib.spiders import CrawlSpider, Rule + from googledir.items import GoogledirItem - rules = ( - Rule(RegexLinkExtractor(allow=('google.com/[A-Z][a-zA-Z_/]+$', ), ), - 'parse_category', - follow=True, - ), - ) + class GoogleDirectorySpider(CrawlSpider): + domain_name = 'google.com' + start_urls = ['http://www.google.com/dirhp'] - def parse_category(self, response): - pass + rules = ( + Rule(RegexLinkExtractor(allow=('google.com/[A-Z][a-zA-Z_/]+$',),), + 'parse_category', + follow=True, + ), + ) - SPIDER = GoogleDirectorySpider() + def parse_category(self, response): + pass + SPIDER = GoogleDirectorySpider() You can try crawling with this little code, by running:: diff --git a/scrapy/trunk/docs/intro/tutorial/tutorial3.rst b/scrapy/trunk/docs/intro/tutorial/tutorial3.rst index c8db65b8c..6a7e4a225 100644 --- a/scrapy/trunk/docs/intro/tutorial/tutorial3.rst +++ b/scrapy/trunk/docs/intro/tutorial/tutorial3.rst @@ -5,19 +5,19 @@ Scraping our data ================= Before going to extraction we need to make a change to our container of scraped -data called ``GoogleItem`` and defined in ``google/items.py`` module:: +data called ``GoogledirItem`` and defined in ``googledir/items.py`` module:: - from scrapy.contrib.item import RobustScrapedItem + from scrapy.contrib.item import RobustScrapedItem - class GoogleItem(RobustScrapedItem): - """Directory website link""" + class GoogledirItem(RobustScrapedItem): + """Directory website link""" - ATTRIBUTES = { - 'guid': basestring, - 'name': basestring, - 'url': basestring, - 'description': basestring, - } + ATTRIBUTES = { + 'guid': basestring, + 'name': basestring, + 'url': basestring, + 'description': basestring, + } Be sure to inherit from RobustScrapedItem, not ScrapedItem. @@ -59,7 +59,21 @@ Another approach could be, for example, to find any ``font`` tags that have that grey colour of the links, but I prefer to use the first one because it wouldn't be so strange if there were other tags with the same colour. -Anyway, having said that, a possible ``parse_category`` could be:: +Anyway, having said that, let's make a possible ``parse_category``: + +First, modify the imports section in the spider code to look like this:: + + import re + + from scrapy.xpath import HtmlXPathSelector + from scrapy.link.extractors import RegexLinkExtractor + from scrapy.contrib.spiders import CrawlSpider, Rule + from scrapy.contrib_exp import adaptors + from googledir.items import GoogledirItem + +Then, put this code in the ``parse_category`` method:: + + from scrapy.contrib_exp import adaptors def parse_category(self, response): # The selector we're going to use in order to extract data from the page @@ -69,15 +83,15 @@ Anyway, having said that, a possible ``parse_category`` could be:: links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') # The list of functions to apply to an attribute before assigning its value - adaptor_pipe = [adaptors.extract, adaptors.Delist(''), adaptors.strip] + adaptor_pipe = [adaptors.extract, adaptors.delist(''), adaptors.strip] adaptor_map = { - 'name': adaptor_pipe, - 'url': adaptor_pipe, - 'description': adaptor_pipe, - } + 'name': adaptor_pipe, + 'url': adaptor_pipe, + 'description': adaptor_pipe, + } for link in links: - item = GoogleItem() + item = GoogledirItem() item.set_adaptors(adaptor_map) item.attribute('name', link.x('a/text()')) @@ -112,7 +126,7 @@ not), and then return it. In this case we used only two adaptors: * ``extract``, which, as you may imagine, extracts data from the XPath nodes you provide, and returns it as a list. -* ``Delist``, which joins the list that the previous adaptor returned into a +* ``delist``, which joins the list that the previous adaptor returned into a string. This adaptor itself is a class, and this is due to the fact that you must specify which delimiter will join the list. That's why we put an instance to this adaptor in the list. diff --git a/scrapy/trunk/docs/intro/tutorial/tutorial4.rst b/scrapy/trunk/docs/intro/tutorial/tutorial4.rst index d8fac6dd1..4c066f545 100644 --- a/scrapy/trunk/docs/intro/tutorial/tutorial4.rst +++ b/scrapy/trunk/docs/intro/tutorial/tutorial4.rst @@ -20,53 +20,54 @@ attributes to that file, in CSV format. Let's see how would our spider end up looking like after applying this change:: - # -*- coding: utf8 -*- - from scrapy.xpath import HtmlXPathSelector - from google.items import GoogleItem - from scrapy.contrib_exp import adaptors - from scrapy.contrib.spiders import CrawlSpider, Rule - from scrapy.link.extractors import RegexLinkExtractor - from scrapy.utils.misc import items_to_csv + # -*- coding: utf8 -*- + import re - class GoogleDirectorySpider(CrawlSpider): - domain_name = 'google.com' - start_urls = ['http://www.google.com/dirhp'] + from scrapy.xpath import HtmlXPathSelector + from scrapy.link.extractors import RegexLinkExtractor + from scrapy.contrib.spiders import CrawlSpider, Rule + from scrapy.contrib_exp import adaptors + from scrapy.utils.misc import items_to_csv + from googledir.items import GoogledirItem - rules = ( - Rule(RegexLinkExtractor(allow=('google.com/[A-Z][a-zA-Z_/]+$', ), ), - 'parse_category', - follow=True, - ), - ) - csv_file = open('scraped_items.csv', 'ab+') + class GoogleDirectorySpider(CrawlSpider): + domain_name = 'google.com' + start_urls = ['http://www.google.com/dirhp'] - def parse_category(self, response): - # The selector we're going to use in order to extract data from the page - hxs = HtmlXPathSelector(response) + rules = ( + Rule(RegexLinkExtractor(allow=('google.com/[A-Z][a-zA-Z_/]+$',),), + 'parse_category', + follow=True, + ), + ) + csv_file = open('scraped_items.csv', 'ab+') + + def parse_category(self, response): + # The selector we're going to use in order to extract data from the page + hxs = HtmlXPathSelector(response) - # The path to website links in directory page - links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + # The path to website links in directory page + links = hxs.x('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') - # The list of functions to apply to an attribute before assigning its value - adaptor_pipe = [adaptors.extract, adaptors.Delist(''), adaptors.strip] - adaptor_map = { - 'name': adaptor_pipe, - 'url': adaptor_pipe, - 'description': adaptor_pipe, - } + # The list of functions to apply to an attribute before assigning its value + adaptor_pipe = [adaptors.extract, adaptors.delist(''), adaptors.strip] + adaptor_map = { + 'name': adaptor_pipe, + 'url': adaptor_pipe, + 'description': adaptor_pipe, + } - for link in links: - item = GoogleItem() - item.set_adaptors(adaptor_map) + for link in links: + item = GoogledirItem() + item.set_adaptors(adaptor_map) - item.attribute('name', link.x('a/text()')) - item.attribute('url', link.x('a/@href')) - item.attribute('description', link.x('font[2]/text()')) - items_to_csv(self.csv_file, [item]) - yield item - - SPIDER = GoogleDirectorySpider() + item.attribute('name', link.x('a/text()')) + item.attribute('url', link.x('a/@href')) + item.attribute('description', link.x('font[2]/text()')) + items_to_csv(self.csv_file, [item]) + yield item +SPIDER = GoogleDirectorySpider() With this code, our spider will crawl over Google's directory, and save each link's name, description, and url to a file called 'scraped_items.csv'::