From bf73002428408778ee71e511e9282925c7180b02 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 28 Apr 2011 02:28:39 -0300 Subject: [PATCH] removed googledir example, replaced by dirbot project on github. updated docs accordingly --- debian/scrapy.examples | 1 - docs/faq.rst | 20 ++-------- docs/index.rst | 9 ++++- docs/intro/examples.rst | 31 ++++++++++++++ docs/intro/tutorial.rst | 17 +++++--- examples/googledir/googledir/__init__.py | 0 examples/googledir/googledir/items.py | 10 ----- examples/googledir/googledir/pipelines.py | 15 ------- examples/googledir/googledir/settings.py | 14 ------- .../googledir/googledir/spiders/__init__.py | 1 - .../googledir/spiders/google_directory.py | 40 ------------------- examples/googledir/scrapy.cfg | 2 - 12 files changed, 52 insertions(+), 108 deletions(-) delete mode 100644 debian/scrapy.examples create mode 100644 docs/intro/examples.rst delete mode 100644 examples/googledir/googledir/__init__.py delete mode 100644 examples/googledir/googledir/items.py delete mode 100644 examples/googledir/googledir/pipelines.py delete mode 100644 examples/googledir/googledir/settings.py delete mode 100644 examples/googledir/googledir/spiders/__init__.py delete mode 100644 examples/googledir/googledir/spiders/google_directory.py delete mode 100644 examples/googledir/scrapy.cfg diff --git a/debian/scrapy.examples b/debian/scrapy.examples deleted file mode 100644 index e39721e20..000000000 --- a/debian/scrapy.examples +++ /dev/null @@ -1 +0,0 @@ -examples/* diff --git a/docs/faq.rst b/docs/faq.rst index 65159a6c2..953d28be6 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -115,24 +115,10 @@ Try changing the default `Accept-Language`_ request header by overriding the .. _Accept-Language: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4 -Where can I find some example code using Scrapy? ------------------------------------------------- +Where can I find some example Scrapy projects? +---------------------------------------------- -Scrapy comes with a built-in, fully functional project to scrape the `Google -Directory`_. You can find it in the `examples/googledir`_ directory of the -Scrapy distribution. - -Also, there's a site for sharing code snippets (spiders, middlewares, -extensions) called `Scrapy snippets`_. - -Finally, you can find some example code for performing not-so-trivial tasks in -the `Scrapy Recipes`_ wiki page. - -.. _Google Directory: http://www.google.com/dirhp -.. _examples/googledir: http://dev.scrapy.org/browser/examples/googledir -.. _Community Spiders: http://dev.scrapy.org/wiki/CommunitySpiders -.. _Scrapy Recipes: http://dev.scrapy.org/wiki/ScrapyRecipes -.. _Scrapy snippets: http://snippets.scrapy.org/ +See :ref:`intro-examples`. Can I run a spider without creating a project? ---------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index e50ffa15b..0e80cc12c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,6 +33,7 @@ First steps intro/overview intro/install intro/tutorial + intro/examples :doc:`intro/overview` Understand what Scrapy is and how it can help you. @@ -43,9 +44,13 @@ First steps :doc:`intro/tutorial` Write your first Scrapy project. +:doc:`intro/examples` + Learn more by playing with a pre-made Scrapy project. -Scraping basics -=============== +.. _section-basics: + +Basic concepts +============== .. toctree:: :hidden: diff --git a/docs/intro/examples.rst b/docs/intro/examples.rst new file mode 100644 index 000000000..cb54563d0 --- /dev/null +++ b/docs/intro/examples.rst @@ -0,0 +1,31 @@ +.. _intro-examples: + +======== +Examples +======== + +The best way to learn is with examples. For this reason, there is an example +Scrapy project named dirbot_, that you can use to play and learn more about +Scrapy. It contains the dmoz spider described in the tutorial. + +This dirbot_ project is available at: https://github.com/scrapy/dirbot + +It contains a README file with a detailed description of the project contents. + +If you're familiar with git, you can checkout the code. Otherwise you can +download a tarball or zip file of the project by clicking on `Downloads`_. + +Other sources for examples +========================== + +1. There is a site for sharing code snippets (spiders, middlewares, extensions) +called `Scrapy snippets`_. + +2. There is also a `Scrapy Recipes`_ page on the wiki that contains code +snippets for performing not-so-trivial tasks. This contains code posted before +the `Scrapy snippets`_ site was lunched. New code should be posted there. + +.. _dirbot: https://github.com/scrapy/dirbot +.. _Downloads: https://github.com/scrapy/dirbot/archives/master +.. _Scrapy Recipes: http://dev.scrapy.org/wiki/ScrapyRecipes +.. _Scrapy snippets: http://snippets.scrapy.org/ diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index e29ac2e7b..26da2638b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -376,8 +376,8 @@ standard dict syntax like:: 'Example title' Spiders are expected to return their scraped data inside -:class:`~scrapy.item.Item` objects, so to actually return the data we've -scraped so far, the code for our Spider should be like this:: +:class:`~scrapy.item.Item` objects. So, in order to returnthe data we've +scraped so far, the final code for our Spider would be like this:: from scrapy.spider import BaseSpider from scrapy.selector import HtmlXPathSelector @@ -404,6 +404,9 @@ scraped so far, the code for our Spider should be like this:: items.append(item) return items +.. note:: You can find a fully-functional variant of this spider in the dirbot_ + project available at https://github.com/scrapy/dirbot + Now doing a crawl on the dmoz.org domain yields ``DmozItem``'s:: [dmoz.org] DEBUG: Scraped DmozItem(desc=[u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n'], link=[u'http://gnosis.cx/TPiP/'], title=[u'Text Processing in Python']) in @@ -427,11 +430,13 @@ placeholder file for Item Pipelines has been set up for you when the project is created, in ``dmoz/pipelines.py``. Though you don't need to implement any item pipeline if you just want to store the scraped items. -Finale -====== +Next steps +========== This tutorial covers only the basics of Scrapy, but there's a lot of other -features not mentioned here. We recommend you continue reading the section -:ref:`topics-index`. +features not mentioned here. We recommend you continue by playing with an +example project (see :ref:`intro-examples`), and then continue with the section +:ref:`section-basics`. .. _JSON: http://en.wikipedia.org/wiki/JSON +.. _dirbot: https://github.com/scrapy/dirbot diff --git a/examples/googledir/googledir/__init__.py b/examples/googledir/googledir/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/examples/googledir/googledir/items.py b/examples/googledir/googledir/items.py deleted file mode 100644 index c5d11b2cd..000000000 --- a/examples/googledir/googledir/items.py +++ /dev/null @@ -1,10 +0,0 @@ -from scrapy.item import Item, Field - -class GoogledirItem(Item): - - name = Field() - url = Field() - description = Field() - - def __str__(self): - return "Google Category: name=%s url=%s" % (self['name'], self['url']) diff --git a/examples/googledir/googledir/pipelines.py b/examples/googledir/googledir/pipelines.py deleted file mode 100644 index b842bb5cf..000000000 --- a/examples/googledir/googledir/pipelines.py +++ /dev/null @@ -1,15 +0,0 @@ -from scrapy.exceptions import DropItem - -class FilterWordsPipeline(object): - """A pipeline for filtering out items which contain certain words in their - description""" - - # put all words in lowercase - words_to_filter = ['politics', 'religion'] - - def process_item(self, item, spider): - for word in self.words_to_filter: - if word in unicode(item['description']).lower(): - raise DropItem("Contains forbidden word: %s" % word) - else: - return item diff --git a/examples/googledir/googledir/settings.py b/examples/googledir/googledir/settings.py deleted file mode 100644 index 6c4d6ef8a..000000000 --- a/examples/googledir/googledir/settings.py +++ /dev/null @@ -1,14 +0,0 @@ -# - Scrapy settings for googledir project - -import googledir - -BOT_NAME = 'googledir' -BOT_VERSION = '1.0' - -SPIDER_MODULES = ['googledir.spiders'] -NEWSPIDER_MODULE = 'googledir.spiders' -DEFAULT_ITEM_CLASS = 'scrapy.item.Item' -USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) - -ITEM_PIPELINES = ['googledir.pipelines.FilterWordsPipeline'] - diff --git a/examples/googledir/googledir/spiders/__init__.py b/examples/googledir/googledir/spiders/__init__.py deleted file mode 100644 index 59d02a55f..000000000 --- a/examples/googledir/googledir/spiders/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Place here all your scrapy spiders diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py deleted file mode 100644 index ceb370e3e..000000000 --- a/examples/googledir/googledir/spiders/google_directory.py +++ /dev/null @@ -1,40 +0,0 @@ -from scrapy.selector import HtmlXPathSelector -from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor -from scrapy.contrib.spiders import CrawlSpider, Rule -from scrapy.contrib.loader import XPathItemLoader -from googledir.items import GoogledirItem - -class GoogleDirectorySpider(CrawlSpider): - - name = 'directory.google.com' - allowed_domains = ['directory.google.com'] - start_urls = ['http://directory.google.com/'] - - rules = ( - Rule(SgmlLinkExtractor(allow='directory.google.com/[A-Z][a-zA-Z_/]+$'), - 'parse_category', - follow=True, - ), - ) - - def parse_category(self, response): - # The main selector we're using to extract data from the page - main_selector = HtmlXPathSelector(response) - - # The XPath to website links in the directory page - xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font' - - # Get a list of (sub) selectors to each website node pointed by the XPath - sub_selectors = main_selector.select(xpath) - - # Iterate over the sub-selectors to extract data for each website - for selector in sub_selectors: - item = GoogledirItem() - - l = XPathItemLoader(item=item, selector=selector) - l.add_xpath('name', 'a/text()') - l.add_xpath('url', 'a/@href') - l.add_xpath('description', 'font[2]/text()') - - # Here we populate the item and yield it - yield l.load_item() diff --git a/examples/googledir/scrapy.cfg b/examples/googledir/scrapy.cfg deleted file mode 100644 index 164e66589..000000000 --- a/examples/googledir/scrapy.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[settings] -default = googledir.settings