diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index c28cdd9ed..c84fefb1e 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -16,7 +16,4 @@ it's properly merged) . Use at your own risk. This documentation is a work in progress. Use at your own risk. -.. toctree:: - :maxdepth: 1 - - djangoitems +*No experimental features at this time* diff --git a/docs/index.rst b/docs/index.rst index 61b053c9a..8203d85b1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -136,6 +136,7 @@ Solving specific problems topics/ubuntu topics/scrapyd topics/jobs + topics/djangoitem :doc:`faq` Get answers to most frequently asked questions. @@ -164,6 +165,9 @@ Solving specific problems :doc:`topics/jobs` Learn how to pause and resume crawls for large spiders. +:doc:`topics/djangoitem` + Write scraped items using Django models. + .. _extending-scrapy: Extending Scrapy diff --git a/docs/news.rst b/docs/news.rst index cdf61e107..e32e6bcb3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -26,6 +26,7 @@ Scrapy changes: - ``USER_AGENT`` spider attribute will no longer work, use ``user_agent`` attribute instead - ``DOWNLOAD_TIMEOUT`` spider attribute will no longer work, use ``download_timeout`` attribute instead - removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library +- promoted :ref:`topics-djangoitem` to main contrib Scrapyd changes: diff --git a/docs/experimental/djangoitems.rst b/docs/topics/djangoitem.rst similarity index 83% rename from docs/experimental/djangoitems.rst rename to docs/topics/djangoitem.rst index c0ff82236..0fba1e81b 100644 --- a/docs/experimental/djangoitems.rst +++ b/docs/topics/djangoitem.rst @@ -1,6 +1,6 @@ .. _topics-djangoitem: -.. module:: scrapy.contrib_exp.djangoitem +.. module:: scrapy.contrib.djangoitem ========== DjangoItem @@ -87,3 +87,12 @@ And we can override the fields of the model with your own:: This is usefull to provide properties to the field, like a default or any other property that your project uses. + +DjangoItem caveats +================== + +DjangoItem is a rather convenient way to integrate Scrapy projects with Django +models, but bear in mind that Django ORM may not scale well if you scrape a lot +of items (ie. millions) with Scrapy. This is because a relational backend is +often not a good choice for a write intensive application (such as a web +crawler), specially if the database is highly normalized and with many indices. diff --git a/scrapy/contrib_exp/djangoitem.py b/scrapy/contrib_exp/djangoitem.py index 571dace5c..1e855b404 100644 --- a/scrapy/contrib_exp/djangoitem.py +++ b/scrapy/contrib_exp/djangoitem.py @@ -1,33 +1,6 @@ -from scrapy.item import Field, Item, ItemMeta +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.contrib_exp.djangoitem` is deprecated, use `scrapy.contrib.djangoitem` instead", + ScrapyDeprecationWarning, stacklevel=2) - -class DjangoItemMeta(ItemMeta): - - def __new__(mcs, class_name, bases, attrs): - cls = super(DjangoItemMeta, mcs).__new__(mcs, class_name, bases, attrs) - cls.fields = cls.fields.copy() - - if cls.django_model: - cls._model_fields = [] - cls._model_meta = cls.django_model._meta - for model_field in cls._model_meta.fields: - if model_field.auto_created == False: - if model_field.name not in cls.fields: - cls.fields[model_field.name] = Field() - cls._model_fields.append(model_field.name) - return cls - - -class DjangoItem(Item): - - __metaclass__ = DjangoItemMeta - - django_model = None - - def save(self, commit=True): - modelargs = dict((k, self.get(k)) for k in self._values - if k in self._model_fields) - model = self.django_model(**modelargs) - if commit: - model.save() - return model +from scrapy.contrib.djangoitem import DjangoItem diff --git a/scrapy/tests/test_djangoitem/__init__.py b/scrapy/tests/test_djangoitem/__init__.py index e3ffb176f..7f413825d 100644 --- a/scrapy/tests/test_djangoitem/__init__.py +++ b/scrapy/tests/test_djangoitem/__init__.py @@ -1,7 +1,7 @@ import os from twisted.trial import unittest -from scrapy.contrib_exp.djangoitem import DjangoItem, Field +from scrapy.contrib.djangoitem import DjangoItem, Field os.environ['DJANGO_SETTINGS_MODULE'] = 'scrapy.tests.test_djangoitem.settings'