PY3: use six.iterkeys, six.iteritems, and tempfile

scrapy/cmdline.py

@@ -71,7 +71,7 @@ def _print_commands(settings, inproject):
     print("  scrapy <command> [options] [args]\n")
     print("Available commands:")
     cmds = _get_commands_dict(settings, inproject)
-    for cmdname, cmdclass in sorted(cmds.iteritems()):
+    for cmdname, cmdclass in sorted(cmds.items()):
         print("  %-13s %s" % (cmdname, cmdclass.short_desc()))
     if not inproject:
         print()

scrapy/commands/check.py

@@ -86,7 +86,7 @@ class Command(ScrapyCommand):
 
         # start checks
         if opts.list:
-            for spider, methods in sorted(contract_reqs.iteritems()):
+            for spider, methods in sorted(contract_reqs.items()):
                 if not methods and not opts.verbose:
                     continue
                 print(spider)

scrapy/contrib/downloadermiddleware/cookies.py

@@ -1,4 +1,5 @@
 import os
+import six
 from collections import defaultdict
 
 from scrapy.exceptions import NotConfigured
@@ -77,7 +78,7 @@ class CookiesMiddleware(object):
     def _get_request_cookies(self, jar, request):
         if isinstance(request.cookies, dict):
             cookie_list = [{'name': k, 'value': v} for k, v in \
-                    request.cookies.iteritems()]
+                    six.iteritems(request.cookies)]
         else:
             cookie_list = request.cookies
 

scrapy/contrib/exporter/__init__.py

@@ -6,6 +6,7 @@ import csv
 import sys
 import pprint
 import marshal
+import six
 from six.moves import cPickle as pickle
 from xml.sax.saxutils import XMLGenerator
 from scrapy.utils.serialize import ScrapyJSONEncoder
@@ -56,9 +57,9 @@ class BaseItemExporter(object):
             include_empty = self.export_empty_fields
         if self.fields_to_export is None:
             if include_empty:
-                field_iter = item.fields.iterkeys()
+                field_iter = six.iterkeys(item.fields)
             else:
-                field_iter = item.iterkeys()
+                field_iter = six.iterkeys(item)
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -248,7 +249,7 @@ class PythonItemExporter(BaseItemExporter):
         return self._to_str_if_unicode(value)
 
     def _serialize_dict(self, value):
-        for key, val in value.iteritems():
+        for key, val in six.iteritems(value):
             yield key, self._serialize_value(val)
 
     def export_item(self, item):

scrapy/contrib/loader/__init__.py

@@ -4,6 +4,7 @@ See documentation in docs/topics/loaders.rst
 
 """
 from collections import defaultdict
+import six
 
 from scrapy.item import Item
 from scrapy.selector import Selector
@@ -39,7 +40,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k,v in value.iteritems():
+            for k, v in six.iteritems(value):
                 self._add_value(k, v)
         else:
             self._add_value(field_name, value)
@@ -49,7 +50,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k,v in value.iteritems():
+            for k, v in six.iteritems(value):
                 self._replace_value(k, v)
         else:
             self._replace_value(field_name, value)

scrapy/contrib/memdebug.py

@@ -5,6 +5,7 @@ See documentation in docs/topics/extensions.rst
 """
 
 import gc
+import six
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -27,7 +28,7 @@ class MemoryDebugger(object):
     def spider_closed(self, spider, reason):
         gc.collect()
         self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
-        for cls, wdict in live_refs.iteritems():
+        for cls, wdict in six.iteritems(live_refs):
             if not wdict:
                 continue
             self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)

scrapy/contrib/pipeline/files.py

@@ -10,6 +10,7 @@ import time
 import urlparse
 from collections import defaultdict
 from cStringIO import StringIO
+import six
 
 from twisted.internet import defer, threads
 
@@ -105,7 +106,7 @@ class S3FilesStore(object):
         key_name = '%s%s' % (self.prefix, path)
         k = b.new_key(key_name)
         if meta:
-            for metakey, metavalue in meta.iteritems():
+            for metakey, metavalue in six.iteritems(meta):
                 k.set_metadata(metakey, str(metavalue))
         h = self.HEADERS.copy()
         if headers:

scrapy/contrib/pipeline/images.py

@@ -6,6 +6,7 @@ See documentation in topics/images.rst
 
 import hashlib
 from cStringIO import StringIO
+import six
 
 from PIL import Image
 
@@ -79,7 +80,7 @@ class ImagesPipeline(FilesPipeline):
         image, buf = self.convert_image(orig_image)
         yield path, image, buf
 
-        for thumb_id, size in self.THUMBS.iteritems():
+        for thumb_id, size in six.iteritems(self.THUMBS):
             thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf

scrapy/contrib_exp/downloadermiddleware/decompression.py

@@ -8,6 +8,7 @@ import zipfile
 import tarfile
 from cStringIO import StringIO
 from tempfile import mktemp
+import six
 
 from scrapy import log
 from scrapy.responsetypes import responsetypes
@@ -71,7 +72,7 @@ class DecompressionMiddleware(object):
         if not response.body:
             return response
 
-        for fmt, func in self._formats.iteritems():
+        for fmt, func in six.iteritems(self._formats):
             new_response = func(response)
             if new_response:
                 log.msg(format='Decompressed response with format: %(responsefmt)s',

scrapy/core/downloader/handlers/__init__.py

@@ -1,6 +1,7 @@
 """Download handlers for different schemes"""
 
 from twisted.internet import defer
+import six
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
@@ -14,7 +15,7 @@ class DownloadHandlers(object):
         self._notconfigured = {}
         handlers = crawler.settings.get('DOWNLOAD_HANDLERS_BASE')
         handlers.update(crawler.settings.get('DOWNLOAD_HANDLERS', {}))
-        for scheme, clspath in handlers.iteritems():
+        for scheme, clspath in six.iteritems(handlers):
             # Allow to disable a handler just like any other
             # component (extension, middleware, etc).
             if clspath is None:

scrapy/http/request/form.py

@@ -7,6 +7,7 @@ See documentation in docs/topics/request-response.rst
 
 import urllib, urlparse
 import lxml.html
+import six
 from scrapy.http.request import Request
 from scrapy.utils.python import unicode_to_str
 
@@ -21,7 +22,7 @@ class FormRequest(Request):
         super(FormRequest, self).__init__(*args, **kwargs)
 
         if formdata:
-            items = formdata.iteritems() if isinstance(formdata, dict) else formdata
+            items = formdata.items() if isinstance(formdata, dict) else formdata
             querystr = _urlencode(items, self.encoding)
             if self.method == 'POST':
                 self.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded')
@@ -106,7 +107,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
         if clickable and clickable[0] not in formdata and not clickable[0] is None:
             values.append(clickable)
 
-    values.extend(formdata.iteritems())
+    values.extend(formdata.items())
     return values
 
 def _value(ele):
@@ -161,7 +162,7 @@ def _get_clickable(clickdata, form):
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
     xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in clickdata.iteritems())
+            u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].name, el[0].value)

scrapy/item.py

@@ -6,6 +6,7 @@ See documentation in docs/topics/item.rst
 
 from pprint import pformat
 from UserDict import DictMixin
+import six
 
 from scrapy.utils.trackref import object_ref
 
@@ -24,7 +25,7 @@ class ItemMeta(type):
     def __new__(mcs, class_name, bases, attrs):
         fields = {}
         new_attrs = {}
-        for n, v in attrs.iteritems():
+        for n, v in six.iteritems(attrs):
             if isinstance(v, Field):
                 fields[n] = v
             else:
@@ -43,7 +44,7 @@ class DictItem(DictMixin, BaseItem):
     def __init__(self, *args, **kwargs):
         self._values = {}
         if args or kwargs:  # avoid creating dict for most common case
-            for k, v in dict(*args, **kwargs).iteritems():
+            for k, v in six.iteritems(dict(*args, **kwargs)):
                 self[k] = v
 
     def __getitem__(self, key):

scrapy/responsetypes.py

@@ -7,6 +7,7 @@ based on different criteria.
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from cStringIO import StringIO
+import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
@@ -34,7 +35,7 @@ class ResponseTypes(object):
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types')
         self.mimetypes.readfp(StringIO(mimedata))
-        for mimetype, cls in self.CLASSES.iteritems():
+        for mimetype, cls in six.iteritems(self.CLASSES):
             self.classes[mimetype] = load_object(cls)
 
     def from_mimetype(self, mimetype):

scrapy/shell.py

@@ -110,7 +110,7 @@ class Shell(object):
 
     def print_help(self):
         self.p("Available Scrapy objects:")
-        for k, v in sorted(self.vars.iteritems()):
+        for k, v in sorted(self.vars.items()):
             if self._is_relevant(v):
                 self.p("  %-10s %s" % (k, v))
         self.p("Useful shortcuts:")

scrapy/spidermanager.py

@@ -4,6 +4,7 @@ spiders
 """
 
 from zope.interface import implements
+import six
 
 from scrapy import signals
 from scrapy.interfaces import ISpiderManager
@@ -48,7 +49,7 @@ class SpiderManager(object):
             return spcls(**spider_kwargs)
 
     def find_by_request(self, request):
-        return [name for name, cls in self._spiders.iteritems()
+        return [name for name, cls in six.iteritems(self._spiders)
             if cls.handles_request(request)]
 
     def list(self):

scrapy/tests/test_commands.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import subprocess
+import tempfile
 from time import sleep
 from os.path import exists, join, abspath
 from shutil import rmtree
@@ -25,7 +26,7 @@ class ProjectTest(unittest.TestCase):
         rmtree(self.temp_path)
 
     def call(self, *new_args, **kwargs):
-        out = os.tmpfile()
+        out = tempfile.TemporaryFile()
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
         return subprocess.call(args, stdout=out, stderr=out, cwd=self.cwd, \
             env=self.env, **kwargs)

scrapy/tests/test_downloadermiddleware_defaultheaders.py

@@ -1,4 +1,5 @@
 from unittest import TestCase
+import six
 
 from scrapy.contrib.downloadermiddleware.defaultheaders import DefaultHeadersMiddleware
 from scrapy.http import Request
@@ -13,7 +14,7 @@ class TestDefaultHeadersMiddleware(TestCase):
         spider = Spider('foo')
         spider.set_crawler(crawler)
         defaults = dict([(k, [v]) for k, v in \
-            crawler.settings.get('DEFAULT_REQUEST_HEADERS').iteritems()])
+            six.iteritems(crawler.settings.get('DEFAULT_REQUEST_HEADERS'))])
         return defaults, spider, DefaultHeadersMiddleware.from_crawler(crawler)
 
     def test_process_request(self):

scrapy/utils/datatypes.py

@@ -6,6 +6,7 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import copy
+import six
 from collections import OrderedDict
 
 
@@ -133,7 +134,7 @@ class MultiValueDict(dict):
                         self.setlistdefault(key, []).append(value)
                 except TypeError:
                     raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
-        for key, value in kwargs.iteritems():
+        for key, value in six.iteritems(kwargs):
             self.setlistdefault(key, []).append(value)
 
 class SiteNode(object):
@@ -200,7 +201,7 @@ class CaselessDict(dict):
         return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     def update(self, seq):
-        seq = seq.iteritems() if isinstance(seq, dict) else seq
+        seq = seq.items() if isinstance(seq, dict) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
         super(CaselessDict, self).update(iseq)
 

scrapy/utils/python.py

@@ -10,6 +10,7 @@ import re
 import inspect
 import weakref
 import errno
+import six
 from functools import partial, wraps
 
 
@@ -232,7 +233,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
-    for k, v in dict(dct_or_tuples).iteritems():
+    for k, v in six.iteritems(dict(dct_or_tuples)):
         k = k.encode(encoding) if isinstance(k, unicode) else k
         if not keys_only:
             v = v.encode(encoding) if isinstance(v, unicode) else v

scrapy/utils/trackref.py

@@ -10,7 +10,7 @@ alias to object in that case).
 """
 
 from __future__ import print_function
-import weakref, os
+import weakref, os, six
 from collections import defaultdict
 from time import time
 from operator import itemgetter
@@ -33,7 +33,7 @@ class object_ref(object):
 def format_live_refs(ignore=NoneType):
     s = "Live References" + os.linesep + os.linesep
     now = time()
-    for cls, wdict in live_refs.iteritems():
+    for cls, wdict in six.iteritems(live_refs):
         if not wdict:
             continue
         if issubclass(cls, ignore):
@@ -47,12 +47,12 @@ def print_live_refs(*a, **kw):
     print(format_live_refs(*a, **kw))
 
 def get_oldest(class_name):
-    for cls, wdict in live_refs.iteritems():
+    for cls, wdict in six.iteritems(live_refs):
         if cls.__name__ == class_name:
             if wdict:
-                return min(wdict.iteritems(), key=itemgetter(1))[0]
+                return min(six.iteritems(wdict), key=itemgetter(1))[0]
 
 def iter_all(class_name):
-    for cls, wdict in live_refs.iteritems():
+    for cls, wdict in six.iteritems(live_refs):
         if cls.__name__ == class_name:
-            return wdict.iterkeys()
+            return six.iterkeys(wdict)

scrapy/xlib/pydispatch/dispatcher.py

@@ -26,7 +26,7 @@ Internal attributes:
 		vs. the original code.)
 """
 from __future__ import generators
-import types, weakref
+import types, weakref, six
 from scrapy.xlib.pydispatch import saferef, robustapply, errors
 
 __author__ = "Patrick K. O'Brien <pobrien@orbtech.com>"
@@ -463,7 +463,7 @@ def _removeOldBackRefs(senderkey, signal, receiver, receivers):
 		found = 0
 		signals = connections.get(signal)
 		if signals is not None:
-			for sig,recs in connections.get(signal,{}).iteritems():
+			for sig, recs in six.iteritems(connections.get(signal,{})):
 				if sig != signal:
 					for rec in recs:
 						if rec is oldReceiver: