use io.BytesIO and cStringIO instead of six.BytesIO as suggested

scrapy/contrib/pipeline/files.py

@@ -11,6 +11,11 @@ from six.moves.urllib.parse import urlparse
 from collections import defaultdict
 import six
 
+try:
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    from io import BytesIO
+
 from twisted.internet import defer, threads
 
 from scrapy import log
@@ -256,7 +261,7 @@ class FilesPipeline(MediaPipeline):
 
     def file_downloaded(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
-        buf = six.BytesIO(response.body)
+        buf = BytesIO(response.body)
         self.store.persist_file(path, buf, info)
         checksum = md5sum(buf)
         return checksum

scrapy/contrib/pipeline/images.py

@@ -7,6 +7,11 @@ See documentation in topics/images.rst
 import hashlib
 import six
 
+try:
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    from io import BytesIO
+
 from PIL import Image
 
 from scrapy.utils.misc import md5sum
@@ -69,7 +74,7 @@ class ImagesPipeline(FilesPipeline):
 
     def get_images(self, response, request, info):
         path = self.file_path(request, response=response, info=info)
-        orig_image = Image.open(six.BytesIO(response.body))
+        orig_image = Image.open(BytesIO(response.body))
 
         width, height = orig_image.size
         if width < self.MIN_WIDTH or height < self.MIN_HEIGHT:
@@ -96,7 +101,7 @@ class ImagesPipeline(FilesPipeline):
             image = image.copy()
             image.thumbnail(size, Image.ANTIALIAS)
 
-        buf = six.BytesIO()
+        buf = BytesIO()
         image.save(buf, 'JPEG')
         return image, buf
 

scrapy/contrib_exp/downloadermiddleware/decompression.py

@@ -7,8 +7,14 @@ import gzip
 import zipfile
 import tarfile
 from tempfile import mktemp
+
 import six
 
+try:
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    from io import BytesIO
+
 from scrapy import log
 from scrapy.responsetypes import responsetypes
 
@@ -26,7 +32,7 @@ class DecompressionMiddleware(object):
         }
 
     def _is_tar(self, response):
-        archive = six.BytesIO(response.body)
+        archive = BytesIO(response.body)
         try:
             tar_file = tarfile.open(name=mktemp(), fileobj=archive)
         except tarfile.ReadError:
@@ -37,7 +43,7 @@ class DecompressionMiddleware(object):
         return response.replace(body=body, cls=respcls)
 
     def _is_zip(self, response):
-        archive = six.BytesIO(response.body)
+        archive = BytesIO(response.body)
         try:
             zip_file = zipfile.ZipFile(archive)
         except zipfile.BadZipfile:
@@ -49,7 +55,7 @@ class DecompressionMiddleware(object):
         return response.replace(body=body, cls=respcls)
 
     def _is_gzip(self, response):
-        archive = six.BytesIO(response.body)
+        archive = BytesIO(response.body)
         try:
             body = gzip.GzipFile(fileobj=archive).read()
         except IOError:

scrapy/core/downloader/handlers/ftp.py

@@ -29,7 +29,7 @@ In case of status 200 request, response.headers will come with two keys:
 """
 
 import re
-import six
+from io import BytesIO
 from six.moves.urllib.parse import urlparse
 
 from twisted.internet import reactor
@@ -42,7 +42,7 @@ from scrapy.responsetypes import responsetypes
 class ReceivedDataProtocol(Protocol):
     def __init__(self, filename=None):
         self.__filename = filename
-        self.body = open(filename, "w") if filename else six.BytesIO()
+        self.body = open(filename, "w") if filename else BytesIO()
         self.size = 0
 
     def dataReceived(self, data):

scrapy/core/downloader/handlers/http11.py

@@ -1,8 +1,8 @@
 """Download handlers for http and https schemes"""
 
 import re
-import six
 
+from io import BytesIO
 from time import time
 from six.moves.urllib.parse import urldefrag
 
@@ -234,7 +234,7 @@ class _ResponseReader(protocol.Protocol):
         self._finished = finished
         self._txresponse = txresponse
         self._request = request
-        self._bodybuf = six.BytesIO()
+        self._bodybuf = BytesIO()
 
     def dataReceived(self, bodyBytes):
         self._bodybuf.write(bodyBytes)

scrapy/responsetypes.py

@@ -6,6 +6,7 @@ based on different criteria.
 
 from mimetypes import MimeTypes
 from pkgutil import get_data
+from io import BytesIO
 import six
 
 from scrapy.http import Response
@@ -33,7 +34,7 @@ class ResponseTypes(object):
         self.classes = {}
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types')
-        self.mimetypes.readfp(six.BytesIO(mimedata))
+        self.mimetypes.readfp(BytesIO(mimedata))
         for mimetype, cls in six.iteritems(self.CLASSES):
             self.classes[mimetype] = load_object(cls)
 

scrapy/tests/test_contrib_exporter.py

@@ -1,5 +1,5 @@
 import unittest, json
-import six
+from io import BytesIO
 from six.moves import cPickle as pickle
 import lxml.etree
 import re
@@ -19,7 +19,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
     def setUp(self):
         self.i = TestItem(name=u'John\xa3', age='22')
-        self.output = six.BytesIO()
+        self.output = BytesIO()
         self.ie = self._get_exporter()
 
     def _get_exporter(self, **kwargs):
@@ -126,7 +126,7 @@ class PickleItemExporterTest(BaseItemExporterTest):
     def test_export_multiple_items(self):
         i1 = TestItem(name='hello', age='world')
         i2 = TestItem(name='bye', age='world')
-        f = six.BytesIO()
+        f = BytesIO()
         ie = PickleItemExporter(f)
         ie.start_exporting()
         ie.export_item(i1)
@@ -151,21 +151,21 @@ class CsvItemExporterTest(BaseItemExporterTest):
         self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
 
     def test_header(self):
-        output = six.BytesIO()
+        output = BytesIO()
         ie = CsvItemExporter(output, fields_to_export=self.i.fields.keys())
         ie.start_exporting()
         ie.export_item(self.i)
         ie.finish_exporting()
         self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n')
 
-        output = six.BytesIO()
+        output = BytesIO()
         ie = CsvItemExporter(output, fields_to_export=['age'])
         ie.start_exporting()
         ie.export_item(self.i)
         ie.finish_exporting()
         self.assertCsvEqual(output.getvalue(), 'age\r\n22\r\n')
 
-        output = six.BytesIO()
+        output = BytesIO()
         ie = CsvItemExporter(output)
         ie.start_exporting()
         ie.export_item(self.i)
@@ -173,7 +173,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
         ie.finish_exporting()
         self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n')
 
-        output = six.BytesIO()
+        output = BytesIO()
         ie = CsvItemExporter(output, include_headers_line=False)
         ie.start_exporting()
         ie.export_item(self.i)
@@ -186,7 +186,7 @@ class CsvItemExporterTest(BaseItemExporterTest):
             friends = Field()
 
         i = TestItem2(name='John', friends=['Mary', 'Paul'])
-        output = six.BytesIO()
+        output = BytesIO()
         ie = CsvItemExporter(output, include_headers_line=False)
         ie.start_exporting()
         ie.export_item(i)
@@ -216,7 +216,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(self.output.getvalue(), expected_value)
 
     def test_multivalued_fields(self):
-        output = six.BytesIO()
+        output = BytesIO()
         item = TestItem(name=[u'John\xa3', u'Doe'])
         ie = XmlItemExporter(output)
         ie.start_exporting()
@@ -226,7 +226,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(output.getvalue(), expected_value)
 
     def test_nested_item(self):
-        output = six.BytesIO()
+        output = BytesIO()
         i1 = TestItem(name=u'foo\xa3hoo', age='22')
         i2 = TestItem(name=u'bar', age=i1)
         i3 = TestItem(name=u'buz', age=i2)
@@ -248,7 +248,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
         self.assertXmlEquivalent(output.getvalue(), expected_value)
 
     def test_nested_list_item(self):
-        output = six.BytesIO()
+        output = BytesIO()
         i1 = TestItem(name=u'foo')
         i2 = TestItem(name=u'bar')
         i3 = TestItem(name=u'buz', age=[i1, i2])

scrapy/tests/test_contrib_feedexport.py

@@ -1,5 +1,5 @@
 import os
-import six
+from io import BytesIO
 from six.moves.urllib.parse import urlparse
 
 from zope.interface.verify import verifyObject
@@ -67,7 +67,7 @@ class FTPFeedStorageTest(unittest.TestCase):
         self.failUnless(os.path.exists(path))
         self.failUnlessEqual(open(path).read(), b"content")
         # again, to check s3 objects are overwritten
-        yield storage.store(six.BytesIO(b"new content"))
+        yield storage.store(BytesIO(b"new content"))
         self.failUnlessEqual(open(path).read(), b"new content")
 
 
@@ -93,7 +93,7 @@ class StdoutFeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_store(self):
-        out = six.BytesIO()
+        out = BytesIO()
         storage = StdoutFeedStorage('stdout:', _stdout=out)
         file = storage.open(Spider("default"))
         file.write(b"content")

scrapy/tests/test_downloadermiddleware_httpcompression.py

@@ -1,4 +1,4 @@
-import six
+from io import BytesIO
 from unittest import TestCase
 from os.path import join, abspath, dirname
 from gzip import GzipFile
@@ -104,7 +104,7 @@ class HttpCompressionTest(TestCase):
             'Content-Type': 'text/html',
             'Content-Encoding': 'gzip',
         }
-        f = six.BytesIO()
+        f = BytesIO()
         plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)
@@ -122,7 +122,7 @@ class HttpCompressionTest(TestCase):
             'Content-Type': 'text/html',
             'Content-Encoding': 'gzip',
         }
-        f = six.BytesIO()
+        f = BytesIO()
         plainbody = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=gb2312">"""
         zf = GzipFile(fileobj=f, mode='wb')
         zf.write(plainbody)

scrapy/tests/test_log.py

@@ -1,4 +1,4 @@
-import six
+from io import BytesIO
 
 from twisted.python import log as txlog, failure
 from twisted.trial import unittest
@@ -21,7 +21,7 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
     encoding = 'utf-8'
 
     def setUp(self):
-        self.f = six.BytesIO()
+        self.f = BytesIO()
         self.log_observer = log.ScrapyFileLogObserver(self.f, self.level, self.encoding)
         self.log_observer.start()
 

scrapy/tests/test_mail.py

@@ -1,6 +1,6 @@
 import unittest
+from io import BytesIO
 
-import six
 from scrapy.mail import MailSender
 
 class MailSenderTest(unittest.TestCase):
@@ -30,7 +30,7 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(msg.get('Content-Type'), 'text/html')
 
     def test_send_attach(self):
-        attach = six.BytesIO()
+        attach = BytesIO()
         attach.write(b'content')
         attach.seek(0)
         attachs = [('attachment', 'text/plain', attach)]

scrapy/tests/test_pipeline_images.py

@@ -3,7 +3,7 @@ import hashlib
 import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
-import six
+from io import BytesIO
 
 from twisted.trial import unittest
 
@@ -201,7 +201,7 @@ class ImagesPipelineTestCaseFields(unittest.TestCase):
 
 
 def _create_image(format, *a, **kw):
-    buf = six.BytesIO()
+    buf = BytesIO()
     Image.new(*a, **kw).save(buf, format)
     buf.seek(0)
     return Image.open(buf)

scrapy/tests/test_spider.py

@@ -2,7 +2,7 @@ import gzip
 import inspect
 import warnings
 from scrapy.utils.trackref import object_ref
-import six
+from io import BytesIO
 
 from twisted.trial import unittest
 
@@ -196,7 +196,7 @@ class SitemapSpiderTest(SpiderTest):
     spider_class = SitemapSpider
 
     BODY = b"SITEMAP"
-    f = six.BytesIO()
+    f = BytesIO()
     g = gzip.GzipFile(fileobj=f, mode='w+b')
     g.write(BODY)
     g.close()

scrapy/tests/test_utils_jsonrpc.py

@@ -1,4 +1,5 @@
-import unittest, json, six
+import unittest, json
+from io import BytesIO
 
 from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \
     JsonRpcError, jsonrpc_errors
@@ -18,7 +19,7 @@ class urllib_mock(object):
     def urlopen(self, url, request):
         self.url = url
         self.request = request
-        return six.BytesIO(self.response)
+        return BytesIO(self.response)
 
 class TestTarget(object):
 

scrapy/utils/gz.py

@@ -1,5 +1,10 @@
 import struct
-import six
+
+try:
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    from io import BytesIO
+
 from gzip import GzipFile
 
 def gunzip(data):
@@ -7,7 +12,7 @@ def gunzip(data):
 
     This is resilient to CRC checksum errors.
     """
-    f = GzipFile(fileobj=six.BytesIO(data))
+    f = GzipFile(fileobj=BytesIO(data))
     output = b''
     chunk = b'.'
     while chunk:

scrapy/utils/iterators.py

@@ -1,5 +1,10 @@
 import re, csv, six
 
+try:
+    from cStringIO import StringIO as BytesIO
+except ImportError:
+    from io import BytesIO
+
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
 from scrapy import log
@@ -47,7 +52,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None):
     def _getrow(csv_r):
         return [str_to_unicode(field, encoding) for field in next(csv_r)]
 
-    lines = six.BytesIO(_body_or_str(obj, unicode=False))
+    lines = BytesIO(_body_or_str(obj, unicode=False))
     if delimiter:
         csv_r = csv.reader(lines, delimiter=delimiter)
     else: