PY3: use six.BytesIO and six.moves.cStringIO

scrapy/contrib/pipeline/files.py

@@ -9,7 +9,6 @@ import rfc822
 import time
 from six.moves.urllib.parse import urlparse
 from collections import defaultdict
-from cStringIO import StringIO
 import six
 
 from twisted.internet import defer, threads
@@ -257,7 +256,7 @@ class FilesPipeline(MediaPipeline):
 
     def file_downloaded(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
-        buf = StringIO(response.body)
+        buf = six.BytesIO(response.body)
         self.store.persist_file(path, buf, info)
         checksum = md5sum(buf)
         return checksum

scrapy/contrib/pipeline/images.py

@@ -5,7 +5,6 @@ See documentation in topics/images.rst
 """
 
 import hashlib
-from cStringIO import StringIO
 import six
 
 from PIL import Image
@@ -70,7 +69,7 @@ class ImagesPipeline(FilesPipeline):
 
     def get_images(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
-        orig_image = Image.open(StringIO(response.body))
+        orig_image = Image.open(six.BytesIO(response.body))
 
         width, height = orig_image.size
         if width < self.MIN_WIDTH or height < self.MIN_HEIGHT:
@@ -97,7 +96,7 @@ class ImagesPipeline(FilesPipeline):
             image = image.copy()
             image.thumbnail(size, Image.ANTIALIAS)
 
-        buf = StringIO()
+        buf = six.BytesIO()
         image.save(buf, 'JPEG')
         return image, buf
 

scrapy/contrib_exp/downloadermiddleware/decompression.py

@@ -6,7 +6,6 @@ import bz2
 import gzip
 import zipfile
 import tarfile
-from cStringIO import StringIO
 from tempfile import mktemp
 import six
 
@@ -27,7 +26,7 @@ class DecompressionMiddleware(object):
         }
 
     def _is_tar(self, response):
-        archive = StringIO(response.body)
+        archive = six.BytesIO(response.body)
         try:
             tar_file = tarfile.open(name=mktemp(), fileobj=archive)
         except tarfile.ReadError:
@@ -38,7 +37,7 @@ class DecompressionMiddleware(object):
         return response.replace(body=body, cls=respcls)
 
     def _is_zip(self, response):
-        archive = StringIO(response.body)
+        archive = six.BytesIO(response.body)
         try:
             zip_file = zipfile.ZipFile(archive)
         except zipfile.BadZipfile:
@@ -50,7 +49,7 @@ class DecompressionMiddleware(object):
         return response.replace(body=body, cls=respcls)
 
     def _is_gzip(self, response):
-        archive = StringIO(response.body)
+        archive = six.BytesIO(response.body)
         try:
             body = gzip.GzipFile(fileobj=archive).read()
         except IOError:

scrapy/core/downloader/handlers/ftp.py

@@ -29,8 +29,8 @@ In case of status 200 request, response.headers will come with two keys:
 """
 
 import re
+import six
 from six.moves.urllib.parse import urlparse
-from cStringIO import StringIO
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed
@@ -42,7 +42,7 @@ from scrapy.responsetypes import responsetypes
 class ReceivedDataProtocol(Protocol):
     def __init__(self, filename=None):
         self.__filename = filename
-        self.body = open(filename, "w") if filename else StringIO()
+        self.body = open(filename, "w") if filename else six.BytesIO()
         self.size = 0
 
     def dataReceived(self, data):

scrapy/core/downloader/handlers/http11.py

@@ -1,9 +1,9 @@
 """Download handlers for http and https schemes"""
 
 import re
+import six
 
 from time import time
-from cStringIO import StringIO
 from six.moves.urllib.parse import urldefrag
 
 from zope.interface import implements
@@ -234,7 +234,7 @@ class _ResponseReader(protocol.Protocol):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
-        self._bodybuf = StringIO()
+        self._bodybuf = six.BytesIO()
 
     def dataReceived(self, bodyBytes):
         self._bodybuf.write(bodyBytes)

scrapy/mail.py

@@ -3,7 +3,7 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
-from cStringIO import StringIO
+from six.moves import cStringIO as StringIO
 import six
 
 from email.utils import COMMASPACE, formatdate

scrapy/responsetypes.py

@@ -6,7 +6,6 @@ based on different criteria.
 
 from mimetypes import MimeTypes
 from pkgutil import get_data
-from cStringIO import StringIO
 import six
 
 from scrapy.http import Response
@@ -34,7 +33,7 @@ class ResponseTypes(object):
         self.classes = {}
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types')
-        self.mimetypes.readfp(StringIO(mimedata))
+        self.mimetypes.readfp(six.BytesIO(mimedata))
         for mimetype, cls in six.iteritems(self.CLASSES):
             self.classes[mimetype] = load_object(cls)
 

scrapy/tests/test_contrib_exporter.py

@@ -1,6 +1,6 @@
 import unittest, json
+import six
 from six.moves import cPickle as pickle
-from cStringIO import StringIO
 import lxml.etree
 import re
 
@@ -19,7 +19,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
     def setUp(self):
         self.i = TestItem(name=u'John\xa3', age='22')
-        self.output = StringIO()
+        self.output = six.BytesIO()
         self.ie = self._get_exporter()
 
     def _get_exporter(self, **kwargs):
@@ -126,7 +126,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
     def test_export_multiple_items(self):
         i1 = TestItem(name='hello', age='world')
         i2 = TestItem(name='bye', age='world')
-        f = StringIO()
+        f = six.BytesIO()
         ie = PickleItemExporter(f)
         ie.start_exporting()
         ie.export_item(i1)
@@ -151,21 +151,21 @@ class CsvItemExporterTest(BaseItemExporterTest):
         self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
 
     def test_header(self):
-        output = StringIO()
+        output = six.BytesIO()
         ie = CsvItemExporter(output, fields_to_export=self.i.fields.keys())
         ie.start_exporting()
         ie.export_item(self.i)
         ie.finish_exporting()
         self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
 
-        output = StringIO()
+        output = six.BytesIO()
         ie = CsvItemExporter(output, fields_to_export=['age'])
         ie.start_exporting()
         ie.export_item(self.i)
         ie.finish_exporting()
         self.assertCsvEqual(output.getvalue(), 'age\r\n22\r\n')
 
-        output = StringIO()
+        output = six.BytesIO()
         ie = CsvItemExporter(output)
         ie.start_exporting()
         ie.export_item(self.i)
@@ -173,7 +173,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
         ie.finish_exporting()
         self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
 
-        output = StringIO()
+        output = six.BytesIO()
         ie = CsvItemExporter(output, include_headers_line=False)
         ie.start_exporting()
         ie.export_item(self.i)
@@ -186,7 +186,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
             friends = Field()
 
         i = TestItem2(name='John', friends=['Mary', 'Paul'])
-        output = StringIO()
+        output = six.BytesIO()
         ie = CsvItemExporter(output, include_headers_line=False)
         ie.start_exporting()
         ie.export_item(i)
@@ -216,7 +216,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
-        output = StringIO()
+        output = six.BytesIO()
         item = TestItem(name=[u'John\xa3', u'Doe'])
         ie = XmlItemExporter(output)
         ie.start_exporting()
@@ -226,7 +226,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(output.getvalue(), expected_value)
 
     def test_nested_item(self):
-        output = StringIO()
+        output = six.BytesIO()
         i1 = TestItem(name=u'foo\xa3hoo', age='22')
         i2 = TestItem(name=u'bar', age=i1)
         i3 = TestItem(name=u'buz', age=i2)
@@ -248,7 +248,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(output.getvalue(), expected_value)
 
     def test_nested_list_item(self):
-        output = StringIO()
+        output = six.BytesIO()
         i1 = TestItem(name=u'foo')
         i2 = TestItem(name=u'bar')
         i3 = TestItem(name=u'buz', age=[i1, i2])

scrapy/tests/test_contrib_feedexport.py

@@ -1,6 +1,6 @@
 import os
+import six
 from six.moves.urllib.parse import urlparse
-from cStringIO import StringIO
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
@@ -62,13 +62,13 @@ class FTPFeedStorageTest(unittest.TestCase):
     def _assert_stores(self, storage, path):
         spider = Spider("default")
         file = storage.open(spider)
-        file.write("content")
+        file.write(b"content")
         yield storage.store(file)
         self.failUnless(os.path.exists(path))
-        self.failUnlessEqual(open(path).read(), "content")
+        self.failUnlessEqual(open(path).read(), b"content")
         # again, to check s3 objects are overwritten
-        yield storage.store(StringIO("new content"))
-        self.failUnlessEqual(open(path).read(), "new content")
+        yield storage.store(six.BytesIO(b"new content"))
+        self.failUnlessEqual(open(path).read(), b"new content")
 
 
 class S3FeedStorageTest(unittest.TestCase):
@@ -93,9 +93,9 @@ class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store(self):
-        out = StringIO()
+        out = six.BytesIO()
         storage = StdoutFeedStorage('stdout:', _stdout=out)
         file = storage.open(Spider("default"))
-        file.write("content")
+        file.write(b"content")
         yield storage.store(file)
-        self.assertEqual(out.getvalue(), "content")
+        self.assertEqual(out.getvalue(), b"content")

scrapy/tests/test_downloadermiddleware_httpcompression.py

@@ -1,6 +1,6 @@
+import six
 from unittest import TestCase
 from os.path import join, abspath, dirname
-from cStringIO import StringIO
 from gzip import GzipFile
 
 from scrapy.spider import Spider
@@ -104,8 +104,8 @@ class HttpCompressionTest(TestCase):
             'Content-Type': 'text/html',
             'Content-Encoding': 'gzip',
         }
-        f = StringIO()
-        plainbody = """<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        f = six.BytesIO()
+        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)
         zf.close()
@@ -122,8 +122,8 @@ class HttpCompressionTest(TestCase):
             'Content-Type': 'text/html',
             'Content-Encoding': 'gzip',
         }
-        f = StringIO()
-        plainbody = """<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
+        f = six.BytesIO()
+        plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)
         zf.close()

scrapy/tests/test_log.py

@@ -1,4 +1,4 @@
-from cStringIO import StringIO
+import six
 
 from twisted.python import log as txlog, failure
 from twisted.trial import unittest
@@ -21,7 +21,7 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
     encoding = 'utf-8'
 
     def setUp(self):
-        self.f = StringIO()
+        self.f = six.BytesIO()
         self.log_observer = log.ScrapyFileLogObserver(self.f, self.level, self.encoding)
         self.log_observer.start()
 

scrapy/tests/test_mail.py

@@ -1,6 +1,6 @@
 import unittest
 
-from cStringIO import StringIO
+import six
 from scrapy.mail import MailSender
 
 class MailSenderTest(unittest.TestCase):
@@ -30,8 +30,8 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(msg.get('Content-Type'), 'text/html')
 
     def test_send_attach(self):
-        attach = StringIO()
-        attach.write('content')
+        attach = six.BytesIO()
+        attach.write(b'content')
         attach.seek(0)
         attachs = [('attachment', 'text/plain', attach)]
 

scrapy/tests/test_pipeline_images.py

@@ -1,9 +1,9 @@
 import os
 import hashlib
 import warnings
-from cStringIO import StringIO
 from tempfile import mkdtemp
 from shutil import rmtree
+import six
 
 from twisted.trial import unittest
 
@@ -201,7 +201,7 @@ class ImagesPipelineTestCaseFields(unittest.TestCase):
 
 
 def _create_image(format, *a, **kw):
-    buf = StringIO()
+    buf = six.BytesIO()
     Image.new(*a, **kw).save(buf, format)
     buf.seek(0)
     return Image.open(buf)

scrapy/tests/test_spider.py

@@ -1,8 +1,8 @@
 import gzip
 import inspect
 import warnings
-from cStringIO import StringIO
 from scrapy.utils.trackref import object_ref
+import six
 
 from twisted.trial import unittest
 
@@ -57,7 +57,7 @@ class XMLFeedSpiderTest(SpiderTest):
     spider_class = XMLFeedSpider
 
     def test_register_namespace(self):
-        body = """<?xml version="1.0" encoding="UTF-8"?>
+        body = b"""<?xml version="1.0" encoding="UTF-8"?>
         <urlset xmlns:x="http://www.google.com/schemas/sitemap/0.84"
                 xmlns:y="http://www.example.com/schemas/extras/1.0">
         <url><x:loc>http://www.example.com/Special-Offers.html</loc><y:updated>2009-08-16</updated><other value="bar" y:custom="fuu"/></url>
@@ -103,7 +103,7 @@ class CSVFeedSpiderTest(SpiderTest):
 
 class CrawlSpiderTest(SpiderTest):
 
-    test_body = """<html><head><title>Page title<title>
+    test_body = b"""<html><head><title>Page title<title>
     <body>
     <p><a href="item/12.html">Item 12</a></p>
     <div class='links'>
@@ -195,8 +195,8 @@ class SitemapSpiderTest(SpiderTest):
 
     spider_class = SitemapSpider
 
-    BODY = "SITEMAP"
-    f = StringIO()
+    BODY = b"SITEMAP"
+    f = six.BytesIO()
     g = gzip.GzipFile(fileobj=f, mode='w+b')
     g.write(BODY)
     g.close()

scrapy/tests/test_utils_jsonrpc.py

@@ -1,5 +1,4 @@
-import unittest, json
-from cStringIO import StringIO
+import unittest, json, six
 
 from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \
     JsonRpcError, jsonrpc_errors
@@ -19,7 +18,7 @@ class urllib_mock(object):
     def urlopen(self, url, request):
         self.url = url
         self.request = request
-        return StringIO(self.response)
+        return six.BytesIO(self.response)
 
 class TestTarget(object):
 

scrapy/tests/test_utils_misc/__init__.py

@@ -1,7 +1,6 @@
 import sys
 import os
 import unittest
-from cStringIO import StringIO
 
 from scrapy.item import Item, Field
 from scrapy.utils.misc import load_object, arg_to_iter, walk_modules

scrapy/utils/gz.py

@@ -1,5 +1,5 @@
 import struct
-from cStringIO import StringIO
+import six
 from gzip import GzipFile
 
 def gunzip(data):
@@ -7,9 +7,9 @@ def gunzip(data):
 
     This is resilient to CRC checksum errors.
     """
-    f = GzipFile(fileobj=StringIO(data))
-    output = ''
-    chunk = '.'
+    f = GzipFile(fileobj=six.BytesIO(data))
+    output = b''
+    chunk = b'.'
     while chunk:
         try:
             chunk = f.read(8196)

scrapy/utils/iterators.py

@@ -1,5 +1,4 @@
-import re, csv
-from cStringIO import StringIO
+import re, csv, six
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
@@ -48,7 +47,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None):
     def _getrow(csv_r):
         return [str_to_unicode(field, encoding) for field in next(csv_r)]
 
-    lines = StringIO(_body_or_str(obj, unicode=False))
+    lines = six.BytesIO(_body_or_str(obj, unicode=False))
     if delimiter:
         csv_r = csv.reader(lines, delimiter=delimiter)
     else:
@@ -68,7 +67,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None):
 
 
 def _body_or_str(obj, unicode=True):
-    assert isinstance(obj, (Response, basestring)), \
+    assert isinstance(obj, (Response, six.string_types)), \
         "obj must be Response or basestring, not %s" % type(obj).__name__
     if isinstance(obj, Response):
         if not unicode:
@@ -77,7 +76,7 @@ def _body_or_str(obj, unicode=True):
             return obj.body_as_unicode()
         else:
             return obj.body.decode('utf-8')
-    elif type(obj) is type(u''):
+    elif type(obj) is six.text_type:
         return obj if unicode else obj.encode('utf-8')
     else:
         return obj.decode('utf-8') if unicode else obj