diff --git a/scrapy/item.py b/scrapy/item.py index 031b80a2d..393302dce 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -4,16 +4,22 @@ Scrapy Item See documentation in docs/topics/item.rst """ -from pprint import pformat -from collections import MutableMapping -from copy import deepcopy - from abc import ABCMeta +from pprint import pformat +from copy import deepcopy +import collections + import six from scrapy.utils.trackref import object_ref +if six.PY3: + MutableMapping = collections.abc.MutableMapping +else: + MutableMapping = collections.MutableMapping + + class BaseItem(object_ref): """Base class for all scraped items.""" pass diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index eb373c501..af39e887f 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,13 +6,20 @@ This module must not depend on any module outside the Standard Library. """ import copy -import six +import collections import warnings -from collections import OrderedDict, Mapping + +import six from scrapy.exceptions import ScrapyDeprecationWarning +if six.PY3: + Mapping = collections.abc.Mapping +else: + Mapping = collections.Mapping + + class MultiValueDictKeyError(KeyError): def __init__(self, *args, **kwargs): warnings.warn( @@ -289,7 +296,7 @@ class MergeDict(object): return self.__copy__() -class LocalCache(OrderedDict): +class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. Older items expires first. diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 60306eacb..2b3548bdd 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import -import re from twisted.internet import reactor, error from twisted.internet.defer import Deferred, DeferredList, maybeDeferred from twisted.python import failure @@ -31,18 +30,16 @@ class RobotsTxtMiddlewareTest(unittest.TestCase): def _get_successful_crawler(self): crawler = self.crawler crawler.settings.set('ROBOTSTXT_OBEY', True) - ROBOTS = re.sub(b'^\s+(?m)', b'', u''' - User-Agent: * - Disallow: /admin/ - Disallow: /static/ - - # taken from https://en.wikipedia.org/robots.txt - Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: - Disallow: /wiki/Käyttäjä: - - User-Agent: UnicödeBöt - Disallow: /some/randome/page.html - '''.encode('utf-8')) + ROBOTS = u""" +User-Agent: * +Disallow: /admin/ +Disallow: /static/ +# taken from https://en.wikipedia.org/robots.txt +Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: +Disallow: /wiki/Käyttäjä: +User-Agent: UnicödeBöt +Disallow: /some/randome/page.html +""".encode('utf-8') response = TextResponse('http://site.local/robots.txt', body=ROBOTS) def return_response(request, spider): deferred = Deferred()