promoted DjangoItem to main contrib

docs/experimental/index.rst

@@ -16,7 +16,4 @@ it's properly merged) . Use at your own risk.
 
    This documentation is a work in progress. Use at your own risk.
 
-.. toctree::
-   :maxdepth: 1
-
-   djangoitems
+*No experimental features at this time*

docs/index.rst

@@ -136,6 +136,7 @@ Solving specific problems
    topics/ubuntu
    topics/scrapyd
    topics/jobs
+   topics/djangoitem
 
 :doc:`faq`
     Get answers to most frequently asked questions.
@@ -164,6 +165,9 @@ Solving specific problems
 :doc:`topics/jobs`
     Learn how to pause and resume crawls for large spiders.
 
+:doc:`topics/djangoitem`
+    Write scraped items using Django models.
+
 .. _extending-scrapy:
 
 Extending Scrapy

docs/news.rst

@@ -26,6 +26,7 @@ Scrapy changes:
 - ``USER_AGENT`` spider attribute will no longer work, use ``user_agent`` attribute instead
 - ``DOWNLOAD_TIMEOUT`` spider attribute will no longer work, use ``download_timeout`` attribute instead
 - removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
+- promoted :ref:`topics-djangoitem` to main contrib
 
 Scrapyd changes:
 

docs/topics/djangoitem.rst

@@ -1,6 +1,6 @@
 .. _topics-djangoitem:
 
-.. module:: scrapy.contrib_exp.djangoitem
+.. module:: scrapy.contrib.djangoitem
 
 ==========
 DjangoItem
@@ -87,3 +87,12 @@ And we can override the fields of the model with your own::
 
 This is usefull to provide properties to the field, like a default or any other
 property that your project uses.
+
+DjangoItem caveats
+==================
+
+DjangoItem is a rather convenient way to integrate Scrapy projects with Django
+models, but bear in mind that Django ORM may not scale well if you scrape a lot
+of items (ie. millions) with Scrapy. This is because a relational backend is
+often not a good choice for a write intensive application (such as a web
+crawler), specially if the database is highly normalized and with many indices.

scrapy/contrib_exp/djangoitem.py

@@ -1,33 +1,6 @@
-from scrapy.item import Field, Item, ItemMeta
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.contrib_exp.djangoitem` is deprecated, use `scrapy.contrib.djangoitem` instead",
+    ScrapyDeprecationWarning, stacklevel=2)
 
-
-class DjangoItemMeta(ItemMeta):
-
-    def  __new__(mcs, class_name, bases, attrs):
-        cls = super(DjangoItemMeta, mcs).__new__(mcs, class_name, bases, attrs)
-        cls.fields = cls.fields.copy()
-
-        if cls.django_model:
-            cls._model_fields = []
-            cls._model_meta = cls.django_model._meta
-            for model_field in cls._model_meta.fields:
-                if model_field.auto_created == False:
-                    if model_field.name not in cls.fields:
-                        cls.fields[model_field.name] = Field()
-                    cls._model_fields.append(model_field.name)
-        return cls
-
-
-class DjangoItem(Item):
-
-    __metaclass__ = DjangoItemMeta
-
-    django_model = None
-
-    def save(self, commit=True):
-        modelargs = dict((k, self.get(k)) for k in self._values
-                         if k in self._model_fields)
-        model = self.django_model(**modelargs)
-        if commit:
-            model.save()
-        return model
+from scrapy.contrib.djangoitem import DjangoItem

scrapy/tests/test_djangoitem/__init__.py

@@ -1,7 +1,7 @@
 import os
 from twisted.trial import unittest
 
-from scrapy.contrib_exp.djangoitem import DjangoItem, Field
+from scrapy.contrib.djangoitem import DjangoItem, Field
 
 os.environ['DJANGO_SETTINGS_MODULE'] = 'scrapy.tests.test_djangoitem.settings'