diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 6a172cbc6..14787950b 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -71,7 +71,7 @@ def _print_commands(settings, inproject): print(" scrapy [options] [args]\n") print("Available commands:") cmds = _get_commands_dict(settings, inproject) - for cmdname, cmdclass in sorted(cmds.iteritems()): + for cmdname, cmdclass in sorted(cmds.items()): print(" %-13s %s" % (cmdname, cmdclass.short_desc())) if not inproject: print() diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index a3d969bbb..40ff9014b 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -86,7 +86,7 @@ class Command(ScrapyCommand): # start checks if opts.list: - for spider, methods in sorted(contract_reqs.iteritems()): + for spider, methods in sorted(contract_reqs.items()): if not methods and not opts.verbose: continue print(spider) diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py index f87952f06..b249f329d 100644 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ b/scrapy/contrib/downloadermiddleware/cookies.py @@ -1,4 +1,5 @@ import os +import six from collections import defaultdict from scrapy.exceptions import NotConfigured @@ -77,7 +78,7 @@ class CookiesMiddleware(object): def _get_request_cookies(self, jar, request): if isinstance(request.cookies, dict): cookie_list = [{'name': k, 'value': v} for k, v in \ - request.cookies.iteritems()] + six.iteritems(request.cookies)] else: cookie_list = request.cookies diff --git a/scrapy/contrib/exporter/__init__.py b/scrapy/contrib/exporter/__init__.py index e7624c983..cc88f8792 100644 --- a/scrapy/contrib/exporter/__init__.py +++ b/scrapy/contrib/exporter/__init__.py @@ -6,6 +6,7 @@ import csv import sys import pprint import marshal +import six from six.moves import cPickle as pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder @@ -56,9 +57,9 @@ class BaseItemExporter(object): include_empty = self.export_empty_fields if self.fields_to_export is None: if include_empty: - field_iter = item.fields.iterkeys() + field_iter = six.iterkeys(item.fields) else: - field_iter = item.iterkeys() + field_iter = six.iterkeys(item) else: if include_empty: field_iter = self.fields_to_export @@ -248,7 +249,7 @@ class PythonItemExporter(BaseItemExporter): return self._to_str_if_unicode(value) def _serialize_dict(self, value): - for key, val in value.iteritems(): + for key, val in six.iteritems(value): yield key, self._serialize_value(val) def export_item(self, item): diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index f13bece41..7be2524b9 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -4,6 +4,7 @@ See documentation in docs/topics/loaders.rst """ from collections import defaultdict +import six from scrapy.item import Item from scrapy.selector import Selector @@ -39,7 +40,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k,v in value.iteritems(): + for k, v in six.iteritems(value): self._add_value(k, v) else: self._add_value(field_name, value) @@ -49,7 +50,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k,v in value.iteritems(): + for k, v in six.iteritems(value): self._replace_value(k, v) else: self._replace_value(field_name, value) diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index e1536d858..263d8ce4c 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -5,6 +5,7 @@ See documentation in docs/topics/extensions.rst """ import gc +import six from scrapy import signals from scrapy.exceptions import NotConfigured @@ -27,7 +28,7 @@ class MemoryDebugger(object): def spider_closed(self, spider, reason): gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider) - for cls, wdict in live_refs.iteritems(): + for cls, wdict in six.iteritems(live_refs): if not wdict: continue self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 0f6d7de08..09a0ca665 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -10,6 +10,7 @@ import time import urlparse from collections import defaultdict from cStringIO import StringIO +import six from twisted.internet import defer, threads @@ -105,7 +106,7 @@ class S3FilesStore(object): key_name = '%s%s' % (self.prefix, path) k = b.new_key(key_name) if meta: - for metakey, metavalue in meta.iteritems(): + for metakey, metavalue in six.iteritems(meta): k.set_metadata(metakey, str(metavalue)) h = self.HEADERS.copy() if headers: diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index be3065179..e955e72bf 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -6,6 +6,7 @@ See documentation in topics/images.rst import hashlib from cStringIO import StringIO +import six from PIL import Image @@ -79,7 +80,7 @@ class ImagesPipeline(FilesPipeline): image, buf = self.convert_image(orig_image) yield path, image, buf - for thumb_id, size in self.THUMBS.iteritems(): + for thumb_id, size in six.iteritems(self.THUMBS): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index d67794d67..6fad5b9b1 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -8,6 +8,7 @@ import zipfile import tarfile from cStringIO import StringIO from tempfile import mktemp +import six from scrapy import log from scrapy.responsetypes import responsetypes @@ -71,7 +72,7 @@ class DecompressionMiddleware(object): if not response.body: return response - for fmt, func in self._formats.iteritems(): + for fmt, func in six.iteritems(self._formats): new_response = func(response) if new_response: log.msg(format='Decompressed response with format: %(responsefmt)s', diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index ff82bffd5..ea0842e62 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,6 +1,7 @@ """Download handlers for different schemes""" from twisted.internet import defer +import six from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -14,7 +15,7 @@ class DownloadHandlers(object): self._notconfigured = {} handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE') handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {})) - for scheme, clspath in handlers.iteritems(): + for scheme, clspath in six.iteritems(handlers): # Allow to disable a handler just like any other # component (extension, middleware, etc). if clspath is None: diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 69088c48b..a4f45a306 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -7,6 +7,7 @@ See documentation in docs/topics/request-response.rst import urllib, urlparse import lxml.html +import six from scrapy.http.request import Request from scrapy.utils.python import unicode_to_str @@ -21,7 +22,7 @@ class FormRequest(Request): super(FormRequest, self).__init__(*args, **kwargs) if formdata: - items = formdata.iteritems() if isinstance(formdata, dict) else formdata + items = formdata.items() if isinstance(formdata, dict) else formdata querystr = _urlencode(items, self.encoding) if self.method == 'POST': self.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded') @@ -106,7 +107,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): if clickable and clickable[0] not in formdata and not clickable[0] is None: values.append(clickable) - values.extend(formdata.iteritems()) + values.extend(formdata.items()) return values def _value(ele): @@ -161,7 +162,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such xpath = u'.//*' + \ - u''.join(u'[@%s="%s"]' % c for c in clickdata.iteritems()) + u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata)) el = form.xpath(xpath) if len(el) == 1: return (el[0].name, el[0].value) diff --git a/scrapy/item.py b/scrapy/item.py index 28b419917..ff1c7363d 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst from pprint import pformat from UserDict import DictMixin +import six from scrapy.utils.trackref import object_ref @@ -24,7 +25,7 @@ class ItemMeta(type): def __new__(mcs, class_name, bases, attrs): fields = {} new_attrs = {} - for n, v in attrs.iteritems(): + for n, v in six.iteritems(attrs): if isinstance(v, Field): fields[n] = v else: @@ -43,7 +44,7 @@ class DictItem(DictMixin, BaseItem): def __init__(self, *args, **kwargs): self._values = {} if args or kwargs: # avoid creating dict for most common case - for k, v in dict(*args, **kwargs).iteritems(): + for k, v in six.iteritems(dict(*args, **kwargs)): self[k] = v def __getitem__(self, key): diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 53f0ae426..ac0559a50 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -7,6 +7,7 @@ based on different criteria. from mimetypes import MimeTypes from pkgutil import get_data from cStringIO import StringIO +import six from scrapy.http import Response from scrapy.utils.misc import load_object @@ -34,7 +35,7 @@ class ResponseTypes(object): self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types') self.mimetypes.readfp(StringIO(mimedata)) - for mimetype, cls in self.CLASSES.iteritems(): + for mimetype, cls in six.iteritems(self.CLASSES): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): diff --git a/scrapy/shell.py b/scrapy/shell.py index 323f55f8d..74eaef40f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -110,7 +110,7 @@ class Shell(object): def print_help(self): self.p("Available Scrapy objects:") - for k, v in sorted(self.vars.iteritems()): + for k, v in sorted(self.vars.items()): if self._is_relevant(v): self.p(" %-10s %s" % (k, v)) self.p("Useful shortcuts:") diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index 9ab570964..ce2e36f31 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -4,6 +4,7 @@ spiders """ from zope.interface import implements +import six from scrapy import signals from scrapy.interfaces import ISpiderManager @@ -48,7 +49,7 @@ class SpiderManager(object): return spcls(**spider_kwargs) def find_by_request(self, request): - return [name for name, cls in self._spiders.iteritems() + return [name for name, cls in six.iteritems(self._spiders) if cls.handles_request(request)] def list(self): diff --git a/scrapy/tests/test_commands.py b/scrapy/tests/test_commands.py index f585ba9fc..08b8fc265 100644 --- a/scrapy/tests/test_commands.py +++ b/scrapy/tests/test_commands.py @@ -1,6 +1,7 @@ import os import sys import subprocess +import tempfile from time import sleep from os.path import exists, join, abspath from shutil import rmtree @@ -25,7 +26,7 @@ class ProjectTest(unittest.TestCase): rmtree(self.temp_path) def call(self, *new_args, **kwargs): - out = os.tmpfile() + out = tempfile.TemporaryFile() args = (sys.executable, '-m', 'scrapy.cmdline') + new_args return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, \ env=self.env, **kwargs) diff --git a/scrapy/tests/test_downloadermiddleware_defaultheaders.py b/scrapy/tests/test_downloadermiddleware_defaultheaders.py index bfe570e2b..b37a02336 100644 --- a/scrapy/tests/test_downloadermiddleware_defaultheaders.py +++ b/scrapy/tests/test_downloadermiddleware_defaultheaders.py @@ -1,4 +1,5 @@ from unittest import TestCase +import six from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request @@ -13,7 +14,7 @@ class TestDefaultHeadersMiddleware(TestCase): spider = Spider('foo') spider.set_crawler(crawler) defaults = dict([(k, [v]) for k, v in \ - crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()]) + six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))]) return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler) def test_process_request(self): diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 75828fb32..097bd1ac9 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -6,6 +6,7 @@ This module must not depend on any module outside the Standard Library. """ import copy +import six from collections import OrderedDict @@ -133,7 +134,7 @@ class MultiValueDict(dict): self.setlistdefault(key, []).append(value) except TypeError: raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary") - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): self.setlistdefault(key, []).append(value) class SiteNode(object): @@ -200,7 +201,7 @@ class CaselessDict(dict): return dict.setdefault(self, self.normkey(key), self.normvalue(def_val)) def update(self, seq): - seq = seq.iteritems() if isinstance(seq, dict) else seq + seq = seq.items() if isinstance(seq, dict) else seq iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq) super(CaselessDict, self).update(iseq) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7b09b6afe..6e03d2c12 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -10,6 +10,7 @@ import re import inspect import weakref import errno +import six from functools import partial, wraps @@ -232,7 +233,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): dict or a list of tuples, like any dict constructor supports. """ d = {} - for k, v in dict(dct_or_tuples).iteritems(): + for k, v in six.iteritems(dict(dct_or_tuples)): k = k.encode(encoding) if isinstance(k, unicode) else k if not keys_only: v = v.encode(encoding) if isinstance(v, unicode) else v diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index b930b95dd..252c40632 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -10,7 +10,7 @@ alias to object in that case). """ from __future__ import print_function -import weakref, os +import weakref, os, six from collections import defaultdict from time import time from operator import itemgetter @@ -33,7 +33,7 @@ class object_ref(object): def format_live_refs(ignore=NoneType): s = "Live References" + os.linesep + os.linesep now = time() - for cls, wdict in live_refs.iteritems(): + for cls, wdict in six.iteritems(live_refs): if not wdict: continue if issubclass(cls, ignore): @@ -47,12 +47,12 @@ def print_live_refs(*a, **kw): print(format_live_refs(*a, **kw)) def get_oldest(class_name): - for cls, wdict in live_refs.iteritems(): + for cls, wdict in six.iteritems(live_refs): if cls.__name__ == class_name: if wdict: - return min(wdict.iteritems(), key=itemgetter(1))[0] + return min(six.iteritems(wdict), key=itemgetter(1))[0] def iter_all(class_name): - for cls, wdict in live_refs.iteritems(): + for cls, wdict in six.iteritems(live_refs): if cls.__name__ == class_name: - return wdict.iterkeys() + return six.iterkeys(wdict) diff --git a/scrapy/xlib/pydispatch/dispatcher.py b/scrapy/xlib/pydispatch/dispatcher.py index 08adff921..f5a9a611b 100644 --- a/scrapy/xlib/pydispatch/dispatcher.py +++ b/scrapy/xlib/pydispatch/dispatcher.py @@ -26,7 +26,7 @@ Internal attributes: vs. the original code.) """ from __future__ import generators -import types, weakref +import types, weakref, six from scrapy.xlib.pydispatch import saferef, robustapply, errors __author__ = "Patrick K. O'Brien " @@ -463,7 +463,7 @@ def _removeOldBackRefs(senderkey, signal, receiver, receivers): found = 0 signals = connections.get(signal) if signals is not None: - for sig,recs in connections.get(signal,{}).iteritems(): + for sig, recs in six.iteritems(connections.get(signal,{})): if sig != signal: for rec in recs: if rec is oldReceiver: