From dbc9b37365bf572777ece68dfe48859bda3f825d Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Fri, 25 Jul 2014 00:32:52 +0800 Subject: [PATCH 1/4] PY3: use MutableMapping instead of DictMixin --- scrapy/item.py | 25 +++++++++++++++++++------ scrapy/tests/test_item.py | 22 ++++++++++++++-------- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index ff1c7363d..ec41b212d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -5,7 +5,8 @@ See documentation in docs/topics/item.rst """ from pprint import pformat -from UserDict import DictMixin +from collections import MutableMapping +from abc import ABCMeta import six from scrapy.utils.trackref import object_ref @@ -20,7 +21,7 @@ class Field(dict): """Container of field metadata""" -class ItemMeta(type): +class ItemMeta(ABCMeta): def __new__(mcs, class_name, bases, attrs): fields = {} @@ -37,7 +38,7 @@ class ItemMeta(type): return cls -class DictItem(DictMixin, BaseItem): +class DictItem(MutableMapping, BaseItem): fields = {} @@ -71,6 +72,19 @@ class DictItem(DictMixin, BaseItem): (name, value)) super(DictItem, self).__setattr__(name, value) + def __len__(self): + return len(self._values) + + def __iter__(self): + for i in getattr(self, "_values", ()): + yield i + + def __hash__(self): + if hasattr(self, "_values"): + return hash(frozenset(self._values.items())) + else: + return 1 + def keys(self): return self._values.keys() @@ -81,6 +95,5 @@ class DictItem(DictMixin, BaseItem): return self.__class__(self) -class Item(DictItem): - - __metaclass__ = ItemMeta +class Item(six.with_metaclass(ItemMeta, DictItem)): + pass diff --git a/scrapy/tests/test_item.py b/scrapy/tests/test_item.py index 2a24fddca..5de77d22e 100644 --- a/scrapy/tests/test_item.py +++ b/scrapy/tests/test_item.py @@ -1,6 +1,7 @@ import unittest from scrapy.item import Item, Field +import six class ItemTest(unittest.TestCase): @@ -52,8 +53,13 @@ class ItemTest(unittest.TestCase): i['name'] = u'John Doe' i['number'] = 123 itemrepr = repr(i) - self.assertEqual(itemrepr, - "{'name': u'John Doe', 'number': 123}") + + if six.PY2: + self.assertEqual(itemrepr, + "{'name': u'John Doe', 'number': 123}") + else: + self.assertEqual(itemrepr, + "{'name': 'John Doe', 'number': 123}") i2 = eval(itemrepr) self.assertEqual(i2['name'], 'John Doe') @@ -106,13 +112,13 @@ class ItemTest(unittest.TestCase): i = TestItem() i['name'] = u'John' - self.assertEqual(i.keys(), ['name']) - self.assertEqual(i.values(), ['John']) + self.assertEqual(list(i.keys()), ['name']) + self.assertEqual(list(i.values()), ['John']) i['keys'] = u'Keys' i['values'] = u'Values' - self.assertSortedEqual(i.keys(), ['keys', 'values', 'name']) - self.assertSortedEqual(i.values(), [u'Keys', u'Values', u'John']) + self.assertSortedEqual(list(i.keys()), ['keys', 'values', 'name']) + self.assertSortedEqual(list(i.values()), [u'Keys', u'Values', u'John']) def test_metaclass_inheritance(self): class BaseItem(Item): @@ -125,8 +131,8 @@ class ItemTest(unittest.TestCase): i = TestItem() i['keys'] = 3 - self.assertEqual(i.keys(), ['keys']) - self.assertEqual(i.values(), [3]) + self.assertEqual(list(i.keys()), ['keys']) + self.assertEqual(list(i.values()), [3]) def test_to_dict(self): class TestItem(Item): From 6d215a1b34b16e758e3a72a3181014f3c4ff00b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 24 Jul 2014 15:37:10 -0300 Subject: [PATCH 2/4] items: Use BaseItem.__hash__ and six.add_metaclass decorator --- scrapy/item.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scrapy/item.py b/scrapy/item.py index ec41b212d..9998010b2 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -76,14 +76,9 @@ class DictItem(MutableMapping, BaseItem): return len(self._values) def __iter__(self): - for i in getattr(self, "_values", ()): - yield i + return iter(self._values) - def __hash__(self): - if hasattr(self, "_values"): - return hash(frozenset(self._values.items())) - else: - return 1 + __hash__ = BaseItem.__hash__ def keys(self): return self._values.keys() @@ -95,5 +90,6 @@ class DictItem(MutableMapping, BaseItem): return self.__class__(self) -class Item(six.with_metaclass(ItemMeta, DictItem)): +@six.add_metaclass(ItemMeta) +class Item(DictItem): pass From 79aba9e314742f702fc73b4c86c7f29e447a913e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 24 Jul 2014 15:37:43 -0300 Subject: [PATCH 3/4] pytest 2.6.0 is on pypi now --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 2bd97321e..e385bfb30 100644 --- a/tox.ini +++ b/tox.ini @@ -5,8 +5,6 @@ [tox] envlist = py27, pypy, precise, trunk, py33, docs -indexserver = - HPK = https://devpi.net/hpk/dev/ [testenv] deps = @@ -51,7 +49,7 @@ deps = w3lib>=1.5 # tests requirements mock - :HPK:pytest>2.5.2 + pytest>=2.6.0 pytest-twisted [testenv:windows] From 0cf035dcbd4116997f079948965e4e577dd4e977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 24 Jul 2014 15:38:04 -0300 Subject: [PATCH 4/4] Add tox env for python 3.4 --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index e385bfb30..1474cce39 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,10 @@ deps = pytest>=2.6.0 pytest-twisted +[testenv:py34] +basepython = python3.4 +deps = {[testenv:py33]deps} + [testenv:windows] commands = bin/runtests.bat []