diff --git a/conftest.py b/conftest.py index 258a670dd..21f152e45 100644 --- a/conftest.py +++ b/conftest.py @@ -2,11 +2,7 @@ import six import pytest from twisted.python import log -from scrapy import optional_features - collect_ignore = ["scrapy/stats.py", "scrapy/project.py"] -if 'django' not in optional_features: - collect_ignore.append("tests/test_djangoitem/models.py") if six.PY3: for line in open('tests/py3-ignores.txt'): diff --git a/docs/index.rst b/docs/index.rst index d8601c9ab..97ec9539d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -154,7 +154,6 @@ Solving specific problems topics/autothrottle topics/benchmarking topics/jobs - topics/djangoitem :doc:`faq` Get answers to most frequently asked questions. @@ -198,9 +197,6 @@ Solving specific problems :doc:`topics/jobs` Learn how to pause and resume crawls for large spiders. -:doc:`topics/djangoitem` - Write scraped items using Django models. - .. _extending-scrapy: Extending Scrapy diff --git a/docs/topics/djangoitem.rst b/docs/topics/djangoitem.rst index 84417dfab..ae40401a7 100644 --- a/docs/topics/djangoitem.rst +++ b/docs/topics/djangoitem.rst @@ -4,146 +4,8 @@ DjangoItem ========== -:class:`DjangoItem` is a class of item that gets its fields definition from a -Django model, you simply create a :class:`DjangoItem` and specify what Django -model it relates to. +DjangoItem has been moved into a separate project. -Besides of getting the model fields defined on your item, :class:`DjangoItem` -provides a method to create and populate a Django model instance with the item -data. +It is hosted at: -Using DjangoItem -================ - -:class:`DjangoItem` works much like ModelForms in Django, you create a subclass -and define its ``django_model`` attribute to be a valid Django model. With this -you will get an item with a field for each Django model field. - -In addition, you can define fields that aren't present in the model and even -override fields that are present in the model defining them in the item. - -Let's see some examples: - -Creating a Django model for the examples:: - - from django.db import models - - class Person(models.Model): - name = models.CharField(max_length=255) - age = models.IntegerField() - -Defining a basic :class:`DjangoItem`:: - - from scrapy.contrib.djangoitem import DjangoItem - - class PersonItem(DjangoItem): - django_model = Person - -:class:`DjangoItem` work just like :class:`~scrapy.item.Item`:: - - >>> p = PersonItem() - >>> p['name'] = 'John' - >>> p['age'] = '22' - -To obtain the Django model from the item, we call the extra method -:meth:`~DjangoItem.save` of the :class:`DjangoItem`:: - - >>> person = p.save() - >>> person.name - 'John' - >>> person.age - '22' - >>> person.id - 1 - -The model is already saved when we call :meth:`~DjangoItem.save`, we -can prevent this by calling it with ``commit=False``. We can use -``commit=False`` in :meth:`~DjangoItem.save` method to obtain an unsaved model:: - - >>> person = p.save(commit=False) - >>> person.name - 'John' - >>> person.age - '22' - >>> person.id - None - -As said before, we can add other fields to the item:: - - import scrapy - from scrapy.contrib.djangoitem import DjangoItem - - class PersonItem(DjangoItem): - django_model = Person - sex = scrapy.Field() - -:: - - >>> p = PersonItem() - >>> p['name'] = 'John' - >>> p['age'] = '22' - >>> p['sex'] = 'M' - -.. note:: fields added to the item won't be taken into account when doing a :meth:`~DjangoItem.save` - -And we can override the fields of the model with your own:: - - class PersonItem(DjangoItem): - django_model = Person - name = scrapy.Field(default='No Name') - -This is useful to provide properties to the field, like a default or any other -property that your project uses. - -DjangoItem caveats -================== - -DjangoItem is a rather convenient way to integrate Scrapy projects with Django -models, but bear in mind that Django ORM may not scale well if you scrape a lot -of items (ie. millions) with Scrapy. This is because a relational backend is -often not a good choice for a write intensive application (such as a web -crawler), specially if the database is highly normalized and with many indices. - -Django settings set up -====================== - -To use the Django models outside the Django application you need to set up the -``DJANGO_SETTINGS_MODULE`` environment variable and --in most cases-- modify -the ``PYTHONPATH`` environment variable to be able to import the settings -module. - -There are many ways to do this depending on your use case and preferences. -Below is detailed one of the simplest ways to do it. - -Suppose your Django project is named ``mysite``, is located in the path -``/home/projects/mysite`` and you have created an app ``myapp`` with the model -``Person``. That means your directory structure is something like this:: - - /home/projects/mysite - ├── manage.py - ├── myapp - │   ├── __init__.py - │   ├── models.py - │   ├── tests.py - │   └── views.py - └── mysite - ├── __init__.py - ├── settings.py - ├── urls.py - └── wsgi.py - -Then you need to add ``/home/projects/mysite`` to the ``PYTHONPATH`` -environment variable and set up the environment variable -``DJANGO_SETTINGS_MODULE`` to ``mysite.settings``. That can be done in your -Scrapy's settings file by adding the lines below:: - - import sys - sys.path.append('/home/projects/mysite') - - import os - os.environ['DJANGO_SETTINGS_MODULE'] = 'mysite.settings' - -Notice that we modify the ``sys.path`` variable instead the ``PYTHONPATH`` -environment variable as we are already within the python runtime. If everything -is right, you should be able to start the ``scrapy shell`` command and import -the model ``Person`` (i.e. ``from myapp.models import Person``). + https://github.com/scrapy/scrapy-djangoitem diff --git a/scrapy/__init__.py b/scrapy/__init__.py index d60b239a0..1f38299d2 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -38,13 +38,6 @@ except ImportError: pass else: optional_features.add('boto') -try: - import django - del django -except ImportError: - pass -else: - optional_features.add('django') from twisted import version as _txv twisted_version = (_txv.major, _txv.minor, _txv.micro) diff --git a/scrapy/contrib/djangoitem.py b/scrapy/contrib/djangoitem.py deleted file mode 100644 index 69f7ef1ac..000000000 --- a/scrapy/contrib/djangoitem.py +++ /dev/null @@ -1,75 +0,0 @@ -from scrapy.item import Field, Item, ItemMeta -from scrapy import optional_features -if 'django' in optional_features: - from django.core.exceptions import ValidationError - - -class DjangoItemMeta(ItemMeta): - - def __new__(mcs, class_name, bases, attrs): - cls = super(DjangoItemMeta, mcs).__new__(mcs, class_name, bases, attrs) - cls.fields = cls.fields.copy() - - if cls.django_model: - cls._model_fields = [] - cls._model_meta = cls.django_model._meta - for model_field in cls._model_meta.fields: - if not model_field.auto_created: - if model_field.name not in cls.fields: - cls.fields[model_field.name] = Field() - cls._model_fields.append(model_field.name) - return cls - - -class DjangoItem(Item): - - __metaclass__ = DjangoItemMeta - - django_model = None - - def __init__(self, *args, **kwargs): - super(DjangoItem, self).__init__(*args, **kwargs) - self._instance = None - self._errors = None - - def save(self, commit=True): - if commit: - self.instance.save() - return self.instance - - def is_valid(self, exclude=None): - self._get_errors(exclude) - return not bool(self._errors) - - def _get_errors(self, exclude=None): - if self._errors is not None: - return self._errors - - self._errors = {} - if exclude is None: - exclude = [] - - try: - self.instance.clean_fields(exclude=exclude) - except ValidationError as e: - self._errors = e.update_error_dict(self._errors) - - try: - self.instance.clean() - except ValidationError as e: - self._errors = e.update_error_dict(self._errors) - - # uniqueness is not checked, because it is faster to check it when - # saving object to database. Just beware, that failed save() - # raises IntegrityError instead of ValidationError. - - return self._errors - errors = property(_get_errors) - - @property - def instance(self): - if self._instance is None: - modelargs = dict((k, self.get(k)) for k in self._values - if k in self._model_fields) - self._instance = self.django_model(**modelargs) - return self._instance diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index fb1daf62f..0e7b10386 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -9,7 +9,6 @@ tests/test_contrib_linkextractors.py tests/test_contrib_loader.py tests/test_crawl.py tests/test_crawler.py -tests/test_djangoitem/__init__.py tests/test_downloader_handlers.py tests/test_downloadermiddleware_ajaxcrawlable.py tests/test_downloadermiddleware_cookies.py diff --git a/tests/test_djangoitem/__init__.py b/tests/test_djangoitem/__init__.py deleted file mode 100644 index 45fd7c470..000000000 --- a/tests/test_djangoitem/__init__.py +++ /dev/null @@ -1,103 +0,0 @@ -import os -from twisted.trial import unittest - -from scrapy.contrib.djangoitem import DjangoItem, Field -from scrapy import optional_features - -os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.test_djangoitem.settings' - -if 'django' in optional_features: - from .models import Person, IdentifiedPerson - - class BasePersonItem(DjangoItem): - django_model = Person - - class NewFieldPersonItem(BasePersonItem): - other = Field() - - class OverrideFieldPersonItem(BasePersonItem): - age = Field() - - class IdentifiedPersonItem(DjangoItem): - django_model = IdentifiedPerson - - -class DjangoItemTest(unittest.TestCase): - - def assertSortedEqual(self, first, second, msg=None): - return self.assertEqual(sorted(first), sorted(second), msg) - - def setUp(self): - if 'django' not in optional_features: - raise unittest.SkipTest("Django is not available") - - def test_base(self): - i = BasePersonItem() - self.assertSortedEqual(i.fields.keys(), ['age', 'name']) - - def test_new_fields(self): - i = NewFieldPersonItem() - self.assertSortedEqual(i.fields.keys(), ['age', 'other', 'name']) - - def test_override_field(self): - i = OverrideFieldPersonItem() - self.assertSortedEqual(i.fields.keys(), ['age', 'name']) - - def test_custom_primary_key_field(self): - """ - Test that if a custom primary key exists, it is - in the field list. - """ - i = IdentifiedPersonItem() - self.assertSortedEqual(i.fields.keys(), ['age', 'identifier', 'name']) - - def test_save(self): - i = BasePersonItem() - self.assertSortedEqual(i.fields.keys(), ['age', 'name']) - - i['name'] = 'John' - i['age'] = '22' - person = i.save(commit=False) - - self.assertEqual(person.name, 'John') - self.assertEqual(person.age, '22') - - def test_override_save(self): - i = OverrideFieldPersonItem() - - i['name'] = 'John' - # it is not obvious that "age" should be saved also, since it was - # redefined in child class - i['age'] = '22' - person = i.save(commit=False) - - self.assertEqual(person.name, 'John') - self.assertEqual(person.age, '22') - - def test_validation(self): - long_name = 'z' * 300 - i = BasePersonItem(name=long_name) - self.assertFalse(i.is_valid()) - self.assertEqual(set(i.errors), set(['age', 'name'])) - i = BasePersonItem(name='John') - self.assertTrue(i.is_valid(exclude=['age'])) - self.assertEqual({}, i.errors) - - # once the item is validated, it does not validate again - i['name'] = long_name - self.assertTrue(i.is_valid()) - - def test_override_validation(self): - i = OverrideFieldPersonItem() - i['name'] = 'John' - self.assertFalse(i.is_valid()) - - i = i = OverrideFieldPersonItem() - i['name'] = 'John' - i['age'] = '22' - self.assertTrue(i.is_valid()) - - def test_default_field_values(self): - i = BasePersonItem() - person = i.save(commit=False) - self.assertEqual(person.name, 'Robot') diff --git a/tests/test_djangoitem/models.py b/tests/test_djangoitem/models.py deleted file mode 100644 index 2beaee752..000000000 --- a/tests/test_djangoitem/models.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.db import models - - -class Person(models.Model): - name = models.CharField(max_length=255, default='Robot') - age = models.IntegerField() - - class Meta: - app_label = 'test_djangoitem' - -class IdentifiedPerson(models.Model): - identifier = models.PositiveIntegerField(primary_key=True) - name = models.CharField(max_length=255) - age = models.IntegerField() - - class Meta: - app_label = 'test_djangoitem' diff --git a/tests/test_djangoitem/settings.py b/tests/test_djangoitem/settings.py deleted file mode 100644 index 1bee92477..000000000 --- a/tests/test_djangoitem/settings.py +++ /dev/null @@ -1,8 +0,0 @@ -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ':memory:', - } -} - -SECRET_KEY = 'top-secret' diff --git a/tox.ini b/tox.ini index 2dff74973..ae2ef9e07 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,6 @@ deps = # Extras boto Pillow - django leveldb -rtests/requirements.txt commands = @@ -26,7 +25,6 @@ deps = Twisted==11.1.0 boto==2.2.2 Pillow<2.0 - django==1.3.1 cssselect==0.9.1 zope.interface==3.6.1 -rtests/requirements.txt