diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index b0b046be5..673cda70d 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -19,6 +19,10 @@ it's not revised so frequently), or documentation which overlaps with existing .. toctree:: :maxdepth: 1 - newitem - newitem-adaptors + topics/index + +.. toctree:: + :maxdepth: 2 + + ref/index diff --git a/docs/experimental/ref/index.rst b/docs/experimental/ref/index.rst new file mode 100644 index 000000000..9ef0d55cf --- /dev/null +++ b/docs/experimental/ref/index.rst @@ -0,0 +1,3 @@ +.. toctree:: + + newitem/index diff --git a/docs/experimental/newitem.rst b/docs/experimental/ref/newitem/fields.rst similarity index 54% rename from docs/experimental/newitem.rst rename to docs/experimental/ref/newitem/fields.rst index f5a5e93e2..6e9a87328 100644 --- a/docs/experimental/newitem.rst +++ b/docs/experimental/ref/newitem/fields.rst @@ -1,66 +1,3 @@ -.. _topics-newitem: - -========= -New Items -========= - -The goal of the scraping process is to obtain scraped items from scraped pages. - -Basic scraped items -=================== - -In Scrapy the items are represented by a :class:`~scrapy.item.ScrapedItem` -(almost an empty class) or any subclass of it. - -To use :class:`~scrapy.item.ScrapedItem` you simply instantiate it and use -instance attributes to store the information. - - >>> from scrapy.item import ScrapedItem - >>> item = ScrapedItem() - >>> item.headline = 'Headline' - >>> item.content = 'Content' - >>> item.published = '2009-07-08' - >>> item - ScrapedItem({'headline': 'Headline', 'content': 'Content', 'published': '2009-07-08'}) - -Or you can use your own class to represent items, just be sure it inherits from -:class:`~scrapy.item.ScrapedItem`. - -More advanced items -=================== - -.. class:: scrapy.contrib_exp.newitem.Item - -Scrapy provides :class:`~scrapy.contrib_exp.newitem.Item` (a subclass of -:class:`~scrapy.item.ScrapedItem`) that works like a form with fields to store -the item's data. - -To use this items you first define the item's fields as class attributes:: - - from scrapy.contrib_exp.newitem import Item - from scrapy.contrib_exp.newitem import fields - - class NewsItem(Item): - headline = fields.TextField() - content = fields.TextField() - published = fields.DateField() - -And then you instantiate the item and assign values to its fields, which will be -converted to the expected Python types depending of their class:: - - >>> item = NewsItem() - >>> item.headline = u'Headline' - >>> item.content = u'Content' - >>> item.published = '2009-07-08' - >>> item - NewsItem({'headline': u'Headline', 'content': u'Content', 'published': datetime.date(2009, 7, 8)}) - -Each field accepts a ``default`` argument, that sets the default value of the field. - -Using this may seen complicated at first, but gives you much power over scraped -data, like assigning defaults for fields that are not present in some pages, -:ref:`topic-newitem-adaptors`, etc. - .. _ref-newitem-fields: =========== @@ -77,35 +14,13 @@ Every ``Field`` class constructor accepts these arguments. ``default`` ----------- -The default value for the field. - -Fields which contain a default value will always return that value when not -set, while fields which don't contain a default value will always return -``None`` when not set:: - - from scrapy.contrib_exp.newitem import Item, fields - - class NewsItem(Item): - content = fields.TextField() - author = fields.TextField(default=u'Myself') - published = fields.DateField() - views = fields.IntegerField(default=0) - - >>> it = NewsItem() - >>> it.content is None - True - >>> it.author - u'Myself' - >>> it.published is None - True - >>> it.views - 0 +The default value for the field. See :ref:`topics-newitem-index-defaults`. Field types =========== These are the available built-in ``Field`` types. See -:ref:`newitem-custom-fields` for info on creating your own field types. +:ref:`ref-newitem-fields-custom-fields` for info on creating your own field types. TextField --------- @@ -172,8 +87,7 @@ TimeField .. _datetime.time: http://docs.python.org/library/datetime.html#datetime.time - -.. _newitem-custom-fields: +.. _ref-newitem-fields-custom-fields: Creating custom fields ====================== @@ -194,7 +108,7 @@ BaseField class :meth:`BaseField.to_python` method is not implemented. The ``default`` argument (if given) must be of the type expected by this - field, or any type that is accepted by the :meth:``BaseField.to_python`` + field, or any type that is accepted by the :meth:`BaseField.to_python` method of this field. For example:: diff --git a/docs/experimental/ref/newitem/index.rst b/docs/experimental/ref/newitem/index.rst new file mode 100644 index 000000000..1e18c730d --- /dev/null +++ b/docs/experimental/ref/newitem/index.rst @@ -0,0 +1,12 @@ +.. _ref-newitems-index: + +Items Reference +=============== + +This is the Items reference, for introductory material see :ref:`topics-newitem-index` + +.. toctree:: + :maxdepth: 1 + + fields + diff --git a/docs/experimental/topics/index.rst b/docs/experimental/topics/index.rst new file mode 100644 index 000000000..6c4ce6f27 --- /dev/null +++ b/docs/experimental/topics/index.rst @@ -0,0 +1,4 @@ +.. toctree:: + + newitem/index + newitem/adaptors diff --git a/docs/experimental/newitem-adaptors.rst b/docs/experimental/topics/newitem/adaptors.rst similarity index 99% rename from docs/experimental/newitem-adaptors.rst rename to docs/experimental/topics/newitem/adaptors.rst index 42e2a6123..b4448de06 100644 --- a/docs/experimental/newitem-adaptors.rst +++ b/docs/experimental/topics/newitem/adaptors.rst @@ -1,4 +1,4 @@ -.. _topic-newitem-adaptors: +.. _topics-newitem-adaptors: ============= Item Adaptors diff --git a/docs/experimental/topics/newitem/index.rst b/docs/experimental/topics/newitem/index.rst new file mode 100644 index 000000000..0d0602eef --- /dev/null +++ b/docs/experimental/topics/newitem/index.rst @@ -0,0 +1,100 @@ +.. _topics-newitem-index: + +.. _topics-newitem-scrapeditem: + +===== +Items +===== + +The goal of the scraping process is to obtain scraped items from scraped pages. + +Basic scraped items +=================== + +In Scrapy the items are represented by a :class:`~scrapy.item.ScrapedItem` +(almost an empty class) or any subclass of it. + +To use :class:`~scrapy.item.ScrapedItem` you simply instantiate it and use +instance attributes to store the information. + + >>> from scrapy.item import ScrapedItem + >>> item = ScrapedItem() + >>> item.headline = 'Headline' + >>> item.content = 'Content' + >>> item.published = '2009-07-08' + >>> item + ScrapedItem({'headline': 'Headline', 'content': 'Content', 'published': '2009-07-08'}) + +Or you can use your own class to represent items, just be sure it inherits from +:class:`~scrapy.item.ScrapedItem`. + +.. _topics-newitem-index-item: + +More advanced items +=================== + +.. class:: scrapy.contrib_exp.newitem.Item + +Scrapy provides :class:`~scrapy.contrib_exp.newitem.Item` (a subclass of +:class:`~scrapy.item.ScrapedItem`) that works like a form with fields to store +the item's data. + +To use this items you first define the item's fields as class attributes:: + + from scrapy.contrib_exp.newitem import Item + from scrapy.contrib_exp.newitem import fields + + class NewsItem(Item): + headline = fields.TextField() + content = fields.TextField() + published = fields.DateField() + +And then you instantiate the item and assign values to its fields, which will be +converted to the expected Python types depending of their class:: + + >>> item = NewsItem() + >>> item.headline = u'Headline' + >>> item.content = u'Content' + >>> item.published = '2009-07-08' + >>> item + NewsItem({'headline': u'Headline', 'content': u'Content', 'published': datetime.date(2009, 7, 8)}) + +Using this may seen complicated at first, but gives you much power over scraped +data, like :ref:`topics-newitem-index-defaults`, +:ref:`topics-newitem-adaptors`, etc. + +.. _topics-newitem-index-defaults: + +Default values for fields +------------------------- + +Each field accepts a ``default`` argument, that sets the default value of the +field. + +Fields which contain a default value will always return that value when not +set, while fields which don't contain a default value will always return +``None`` when not set: + +.. code-block:: python + + from scrapy.contrib_exp.newitem import Item, fields + + class NewsItem(Item): + headline = fields.TextField() + content = fields.TextField() + published = fields.DateField() + author = fields.TextField(default=u'Myself') + views = fields.IntegerField(default=0) + +.. code-block:: python + + >>> it = NewsItem() + >>> it.content is None + True + >>> it.author + u'Myself' + >>> it.published is None + True + >>> it.views + 0 + diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 4fbfe2870..ceb8cb0c7 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -57,7 +57,7 @@ RobustScrapedItems .. warning:: RobustScapedItems are deprecated and will be replaced by the :ref:`New item - API ` (still in development). + API ` (still in development). .. module:: scrapy.contrib.item :synopsis: Objects for storing scraped data