diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 93cf9621b..db8cf8b76 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -11,6 +11,11 @@ from six.moves.urllib.parse import urlparse from collections import defaultdict import six +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from twisted.internet import defer, threads from scrapy import log @@ -256,7 +261,7 @@ class FilesPipeline(MediaPipeline): def file_downloaded(self, response, request, info): path = self.file_path(request, response=response, info=info) - buf = six.BytesIO(response.body) + buf = BytesIO(response.body) self.store.persist_file(path, buf, info) checksum = md5sum(buf) return checksum diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index ed29b6f54..9c1a54455 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -7,6 +7,11 @@ See documentation in topics/images.rst import hashlib import six +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from PIL import Image from scrapy.utils.misc import md5sum @@ -69,7 +74,7 @@ class ImagesPipeline(FilesPipeline): def get_images(self, response, request, info): path = self.file_path(request, response=response, info=info) - orig_image = Image.open(six.BytesIO(response.body)) + orig_image = Image.open(BytesIO(response.body)) width, height = orig_image.size if width < self.MIN_WIDTH or height < self.MIN_HEIGHT: @@ -96,7 +101,7 @@ class ImagesPipeline(FilesPipeline): image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - buf = six.BytesIO() + buf = BytesIO() image.save(buf, 'JPEG') return image, buf diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index f6b47eaa1..c08f50b5f 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -7,8 +7,14 @@ import gzip import zipfile import tarfile from tempfile import mktemp + import six +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from scrapy import log from scrapy.responsetypes import responsetypes @@ -26,7 +32,7 @@ class DecompressionMiddleware(object): } def _is_tar(self, response): - archive = six.BytesIO(response.body) + archive = BytesIO(response.body) try: tar_file = tarfile.open(name=mktemp(), fileobj=archive) except tarfile.ReadError: @@ -37,7 +43,7 @@ class DecompressionMiddleware(object): return response.replace(body=body, cls=respcls) def _is_zip(self, response): - archive = six.BytesIO(response.body) + archive = BytesIO(response.body) try: zip_file = zipfile.ZipFile(archive) except zipfile.BadZipfile: @@ -49,7 +55,7 @@ class DecompressionMiddleware(object): return response.replace(body=body, cls=respcls) def _is_gzip(self, response): - archive = six.BytesIO(response.body) + archive = BytesIO(response.body) try: body = gzip.GzipFile(fileobj=archive).read() except IOError: diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 1fd763b40..6ac02cc2b 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -29,7 +29,7 @@ In case of status 200 request, response.headers will come with two keys: """ import re -import six +from io import BytesIO from six.moves.urllib.parse import urlparse from twisted.internet import reactor @@ -42,7 +42,7 @@ from scrapy.responsetypes import responsetypes class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): self.__filename = filename - self.body = open(filename, "w") if filename else six.BytesIO() + self.body = open(filename, "w") if filename else BytesIO() self.size = 0 def dataReceived(self, data): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 47a4fed31..b803af1dc 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,8 +1,8 @@ """Download handlers for http and https schemes""" import re -import six +from io import BytesIO from time import time from six.moves.urllib.parse import urldefrag @@ -234,7 +234,7 @@ class _ResponseReader(protocol.Protocol): self._finished = finished self._txresponse = txresponse self._request = request - self._bodybuf = six.BytesIO() + self._bodybuf = BytesIO() def dataReceived(self, bodyBytes): self._bodybuf.write(bodyBytes) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index f8cd5b4ea..16479896f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -6,6 +6,7 @@ based on different criteria. from mimetypes import MimeTypes from pkgutil import get_data +from io import BytesIO import six from scrapy.http import Response @@ -33,7 +34,7 @@ class ResponseTypes(object): self.classes = {} self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types') - self.mimetypes.readfp(six.BytesIO(mimedata)) + self.mimetypes.readfp(BytesIO(mimedata)) for mimetype, cls in six.iteritems(self.CLASSES): self.classes[mimetype] = load_object(cls) diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index fc5dd82b5..9092007e5 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,5 +1,5 @@ import unittest, json -import six +from io import BytesIO from six.moves import cPickle as pickle import lxml.etree import re @@ -19,7 +19,7 @@ class BaseItemExporterTest(unittest.TestCase): def setUp(self): self.i = TestItem(name=u'John\xa3', age='22') - self.output = six.BytesIO() + self.output = BytesIO() self.ie = self._get_exporter() def _get_exporter(self, **kwargs): @@ -126,7 +126,7 @@ class PickleItemExporterTest(BaseItemExporterTest): def test_export_multiple_items(self): i1 = TestItem(name='hello', age='world') i2 = TestItem(name='bye', age='world') - f = six.BytesIO() + f = BytesIO() ie = PickleItemExporter(f) ie.start_exporting() ie.export_item(i1) @@ -151,21 +151,21 @@ class CsvItemExporterTest(BaseItemExporterTest): self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') def test_header(self): - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, fields_to_export=self.i.fields.keys()) ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, fields_to_export=['age']) ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() self.assertCsvEqual(output.getvalue(), 'age\r\n22\r\n') - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output) ie.start_exporting() ie.export_item(self.i) @@ -173,7 +173,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.finish_exporting() self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, include_headers_line=False) ie.start_exporting() ie.export_item(self.i) @@ -186,7 +186,7 @@ class CsvItemExporterTest(BaseItemExporterTest): friends = Field() i = TestItem2(name='John', friends=['Mary', 'Paul']) - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, include_headers_line=False) ie.start_exporting() ie.export_item(i) @@ -216,7 +216,7 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): - output = six.BytesIO() + output = BytesIO() item = TestItem(name=[u'John\xa3', u'Doe']) ie = XmlItemExporter(output) ie.start_exporting() @@ -226,7 +226,7 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(output.getvalue(), expected_value) def test_nested_item(self): - output = six.BytesIO() + output = BytesIO() i1 = TestItem(name=u'foo\xa3hoo', age='22') i2 = TestItem(name=u'bar', age=i1) i3 = TestItem(name=u'buz', age=i2) @@ -248,7 +248,7 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(output.getvalue(), expected_value) def test_nested_list_item(self): - output = six.BytesIO() + output = BytesIO() i1 = TestItem(name=u'foo') i2 = TestItem(name=u'bar') i3 = TestItem(name=u'buz', age=[i1, i2]) diff --git a/scrapy/tests/test_contrib_feedexport.py b/scrapy/tests/test_contrib_feedexport.py index 925b277b3..bf4943bfa 100644 --- a/scrapy/tests/test_contrib_feedexport.py +++ b/scrapy/tests/test_contrib_feedexport.py @@ -1,5 +1,5 @@ import os -import six +from io import BytesIO from six.moves.urllib.parse import urlparse from zope.interface.verify import verifyObject @@ -67,7 +67,7 @@ class FTPFeedStorageTest(unittest.TestCase): self.failUnless(os.path.exists(path)) self.failUnlessEqual(open(path).read(), b"content") # again, to check s3 objects are overwritten - yield storage.store(six.BytesIO(b"new content")) + yield storage.store(BytesIO(b"new content")) self.failUnlessEqual(open(path).read(), b"new content") @@ -93,7 +93,7 @@ class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - out = six.BytesIO() + out = BytesIO() storage = StdoutFeedStorage('stdout:', _stdout=out) file = storage.open(Spider("default")) file.write(b"content") diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index e37d43d7d..8a0e75d90 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -1,4 +1,4 @@ -import six +from io import BytesIO from unittest import TestCase from os.path import join, abspath, dirname from gzip import GzipFile @@ -104,7 +104,7 @@ class HttpCompressionTest(TestCase): 'Content-Type': 'text/html', 'Content-Encoding': 'gzip', } - f = six.BytesIO() + f = BytesIO() plainbody = b"""