From 0786e84a33155ebc8d8d3502e3a7f3060b86a4ec Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Tue, 15 Jul 2014 20:51:12 +0800 Subject: [PATCH] use io.BytesIO and cStringIO instead of six.BytesIO as suggested --- scrapy/contrib/pipeline/files.py | 7 +++++- scrapy/contrib/pipeline/images.py | 9 ++++++-- .../downloadermiddleware/decompression.py | 12 +++++++--- scrapy/core/downloader/handlers/ftp.py | 4 ++-- scrapy/core/downloader/handlers/http11.py | 4 ++-- scrapy/responsetypes.py | 3 ++- scrapy/tests/test_contrib_exporter.py | 22 +++++++++---------- scrapy/tests/test_contrib_feedexport.py | 6 ++--- ...st_downloadermiddleware_httpcompression.py | 6 ++--- scrapy/tests/test_log.py | 4 ++-- scrapy/tests/test_mail.py | 4 ++-- scrapy/tests/test_pipeline_images.py | 4 ++-- scrapy/tests/test_spider.py | 4 ++-- scrapy/tests/test_utils_jsonrpc.py | 5 +++-- scrapy/utils/gz.py | 9 ++++++-- scrapy/utils/iterators.py | 7 +++++- 16 files changed, 69 insertions(+), 41 deletions(-) diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 93cf9621b..db8cf8b76 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -11,6 +11,11 @@ from six.moves.urllib.parse import urlparse from collections import defaultdict import six +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from twisted.internet import defer, threads from scrapy import log @@ -256,7 +261,7 @@ class FilesPipeline(MediaPipeline): def file_downloaded(self, response, request, info): path = self.file_path(request, response=response, info=info) - buf = six.BytesIO(response.body) + buf = BytesIO(response.body) self.store.persist_file(path, buf, info) checksum = md5sum(buf) return checksum diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index ed29b6f54..9c1a54455 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -7,6 +7,11 @@ See documentation in topics/images.rst import hashlib import six +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from PIL import Image from scrapy.utils.misc import md5sum @@ -69,7 +74,7 @@ class ImagesPipeline(FilesPipeline): def get_images(self, response, request, info): path = self.file_path(request, response=response, info=info) - orig_image = Image.open(six.BytesIO(response.body)) + orig_image = Image.open(BytesIO(response.body)) width, height = orig_image.size if width < self.MIN_WIDTH or height < self.MIN_HEIGHT: @@ -96,7 +101,7 @@ class ImagesPipeline(FilesPipeline): image = image.copy() image.thumbnail(size, Image.ANTIALIAS) - buf = six.BytesIO() + buf = BytesIO() image.save(buf, 'JPEG') return image, buf diff --git a/scrapy/contrib_exp/downloadermiddleware/decompression.py b/scrapy/contrib_exp/downloadermiddleware/decompression.py index f6b47eaa1..c08f50b5f 100644 --- a/scrapy/contrib_exp/downloadermiddleware/decompression.py +++ b/scrapy/contrib_exp/downloadermiddleware/decompression.py @@ -7,8 +7,14 @@ import gzip import zipfile import tarfile from tempfile import mktemp + import six +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from scrapy import log from scrapy.responsetypes import responsetypes @@ -26,7 +32,7 @@ class DecompressionMiddleware(object): } def _is_tar(self, response): - archive = six.BytesIO(response.body) + archive = BytesIO(response.body) try: tar_file = tarfile.open(name=mktemp(), fileobj=archive) except tarfile.ReadError: @@ -37,7 +43,7 @@ class DecompressionMiddleware(object): return response.replace(body=body, cls=respcls) def _is_zip(self, response): - archive = six.BytesIO(response.body) + archive = BytesIO(response.body) try: zip_file = zipfile.ZipFile(archive) except zipfile.BadZipfile: @@ -49,7 +55,7 @@ class DecompressionMiddleware(object): return response.replace(body=body, cls=respcls) def _is_gzip(self, response): - archive = six.BytesIO(response.body) + archive = BytesIO(response.body) try: body = gzip.GzipFile(fileobj=archive).read() except IOError: diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 1fd763b40..6ac02cc2b 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -29,7 +29,7 @@ In case of status 200 request, response.headers will come with two keys: """ import re -import six +from io import BytesIO from six.moves.urllib.parse import urlparse from twisted.internet import reactor @@ -42,7 +42,7 @@ from scrapy.responsetypes import responsetypes class ReceivedDataProtocol(Protocol): def __init__(self, filename=None): self.__filename = filename - self.body = open(filename, "w") if filename else six.BytesIO() + self.body = open(filename, "w") if filename else BytesIO() self.size = 0 def dataReceived(self, data): diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 47a4fed31..b803af1dc 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,8 +1,8 @@ """Download handlers for http and https schemes""" import re -import six +from io import BytesIO from time import time from six.moves.urllib.parse import urldefrag @@ -234,7 +234,7 @@ class _ResponseReader(protocol.Protocol): self._finished = finished self._txresponse = txresponse self._request = request - self._bodybuf = six.BytesIO() + self._bodybuf = BytesIO() def dataReceived(self, bodyBytes): self._bodybuf.write(bodyBytes) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index f8cd5b4ea..16479896f 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -6,6 +6,7 @@ based on different criteria. from mimetypes import MimeTypes from pkgutil import get_data +from io import BytesIO import six from scrapy.http import Response @@ -33,7 +34,7 @@ class ResponseTypes(object): self.classes = {} self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types') - self.mimetypes.readfp(six.BytesIO(mimedata)) + self.mimetypes.readfp(BytesIO(mimedata)) for mimetype, cls in six.iteritems(self.CLASSES): self.classes[mimetype] = load_object(cls) diff --git a/scrapy/tests/test_contrib_exporter.py b/scrapy/tests/test_contrib_exporter.py index fc5dd82b5..9092007e5 100644 --- a/scrapy/tests/test_contrib_exporter.py +++ b/scrapy/tests/test_contrib_exporter.py @@ -1,5 +1,5 @@ import unittest, json -import six +from io import BytesIO from six.moves import cPickle as pickle import lxml.etree import re @@ -19,7 +19,7 @@ class BaseItemExporterTest(unittest.TestCase): def setUp(self): self.i = TestItem(name=u'John\xa3', age='22') - self.output = six.BytesIO() + self.output = BytesIO() self.ie = self._get_exporter() def _get_exporter(self, **kwargs): @@ -126,7 +126,7 @@ class PickleItemExporterTest(BaseItemExporterTest): def test_export_multiple_items(self): i1 = TestItem(name='hello', age='world') i2 = TestItem(name='bye', age='world') - f = six.BytesIO() + f = BytesIO() ie = PickleItemExporter(f) ie.start_exporting() ie.export_item(i1) @@ -151,21 +151,21 @@ class CsvItemExporterTest(BaseItemExporterTest): self.assertCsvEqual(self.output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') def test_header(self): - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, fields_to_export=self.i.fields.keys()) ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n') - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, fields_to_export=['age']) ie.start_exporting() ie.export_item(self.i) ie.finish_exporting() self.assertCsvEqual(output.getvalue(), 'age\r\n22\r\n') - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output) ie.start_exporting() ie.export_item(self.i) @@ -173,7 +173,7 @@ class CsvItemExporterTest(BaseItemExporterTest): ie.finish_exporting() self.assertCsvEqual(output.getvalue(), 'age,name\r\n22,John\xc2\xa3\r\n22,John\xc2\xa3\r\n') - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, include_headers_line=False) ie.start_exporting() ie.export_item(self.i) @@ -186,7 +186,7 @@ class CsvItemExporterTest(BaseItemExporterTest): friends = Field() i = TestItem2(name='John', friends=['Mary', 'Paul']) - output = six.BytesIO() + output = BytesIO() ie = CsvItemExporter(output, include_headers_line=False) ie.start_exporting() ie.export_item(i) @@ -216,7 +216,7 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(self.output.getvalue(), expected_value) def test_multivalued_fields(self): - output = six.BytesIO() + output = BytesIO() item = TestItem(name=[u'John\xa3', u'Doe']) ie = XmlItemExporter(output) ie.start_exporting() @@ -226,7 +226,7 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(output.getvalue(), expected_value) def test_nested_item(self): - output = six.BytesIO() + output = BytesIO() i1 = TestItem(name=u'foo\xa3hoo', age='22') i2 = TestItem(name=u'bar', age=i1) i3 = TestItem(name=u'buz', age=i2) @@ -248,7 +248,7 @@ class XmlItemExporterTest(BaseItemExporterTest): self.assertXmlEquivalent(output.getvalue(), expected_value) def test_nested_list_item(self): - output = six.BytesIO() + output = BytesIO() i1 = TestItem(name=u'foo') i2 = TestItem(name=u'bar') i3 = TestItem(name=u'buz', age=[i1, i2]) diff --git a/scrapy/tests/test_contrib_feedexport.py b/scrapy/tests/test_contrib_feedexport.py index 925b277b3..bf4943bfa 100644 --- a/scrapy/tests/test_contrib_feedexport.py +++ b/scrapy/tests/test_contrib_feedexport.py @@ -1,5 +1,5 @@ import os -import six +from io import BytesIO from six.moves.urllib.parse import urlparse from zope.interface.verify import verifyObject @@ -67,7 +67,7 @@ class FTPFeedStorageTest(unittest.TestCase): self.failUnless(os.path.exists(path)) self.failUnlessEqual(open(path).read(), b"content") # again, to check s3 objects are overwritten - yield storage.store(six.BytesIO(b"new content")) + yield storage.store(BytesIO(b"new content")) self.failUnlessEqual(open(path).read(), b"new content") @@ -93,7 +93,7 @@ class StdoutFeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - out = six.BytesIO() + out = BytesIO() storage = StdoutFeedStorage('stdout:', _stdout=out) file = storage.open(Spider("default")) file.write(b"content") diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index e37d43d7d..8a0e75d90 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -1,4 +1,4 @@ -import six +from io import BytesIO from unittest import TestCase from os.path import join, abspath, dirname from gzip import GzipFile @@ -104,7 +104,7 @@ class HttpCompressionTest(TestCase): 'Content-Type': 'text/html', 'Content-Encoding': 'gzip', } - f = six.BytesIO() + f = BytesIO() plainbody = b"""Some page""" zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) @@ -122,7 +122,7 @@ class HttpCompressionTest(TestCase): 'Content-Type': 'text/html', 'Content-Encoding': 'gzip', } - f = six.BytesIO() + f = BytesIO() plainbody = b"""Some page""" zf = GzipFile(fileobj=f, mode='wb') zf.write(plainbody) diff --git a/scrapy/tests/test_log.py b/scrapy/tests/test_log.py index afe6fe1dd..f0b57b806 100644 --- a/scrapy/tests/test_log.py +++ b/scrapy/tests/test_log.py @@ -1,4 +1,4 @@ -import six +from io import BytesIO from twisted.python import log as txlog, failure from twisted.trial import unittest @@ -21,7 +21,7 @@ class ScrapyFileLogObserverTest(unittest.TestCase): encoding = 'utf-8' def setUp(self): - self.f = six.BytesIO() + self.f = BytesIO() self.log_observer = log.ScrapyFileLogObserver(self.f, self.level, self.encoding) self.log_observer.start() diff --git a/scrapy/tests/test_mail.py b/scrapy/tests/test_mail.py index 392568f38..58d44bdb3 100644 --- a/scrapy/tests/test_mail.py +++ b/scrapy/tests/test_mail.py @@ -1,6 +1,6 @@ import unittest +from io import BytesIO -import six from scrapy.mail import MailSender class MailSenderTest(unittest.TestCase): @@ -30,7 +30,7 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(msg.get('Content-Type'), 'text/html') def test_send_attach(self): - attach = six.BytesIO() + attach = BytesIO() attach.write(b'content') attach.seek(0) attachs = [('attachment', 'text/plain', attach)] diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index fa5c29aa7..a6f161128 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -3,7 +3,7 @@ import hashlib import warnings from tempfile import mkdtemp from shutil import rmtree -import six +from io import BytesIO from twisted.trial import unittest @@ -201,7 +201,7 @@ class ImagesPipelineTestCaseFields(unittest.TestCase): def _create_image(format, *a, **kw): - buf = six.BytesIO() + buf = BytesIO() Image.new(*a, **kw).save(buf, format) buf.seek(0) return Image.open(buf) diff --git a/scrapy/tests/test_spider.py b/scrapy/tests/test_spider.py index 22716cdb1..903eff7b1 100644 --- a/scrapy/tests/test_spider.py +++ b/scrapy/tests/test_spider.py @@ -2,7 +2,7 @@ import gzip import inspect import warnings from scrapy.utils.trackref import object_ref -import six +from io import BytesIO from twisted.trial import unittest @@ -196,7 +196,7 @@ class SitemapSpiderTest(SpiderTest): spider_class = SitemapSpider BODY = b"SITEMAP" - f = six.BytesIO() + f = BytesIO() g = gzip.GzipFile(fileobj=f, mode='w+b') g.write(BODY) g.close() diff --git a/scrapy/tests/test_utils_jsonrpc.py b/scrapy/tests/test_utils_jsonrpc.py index 26e77dd58..8902e9efa 100644 --- a/scrapy/tests/test_utils_jsonrpc.py +++ b/scrapy/tests/test_utils_jsonrpc.py @@ -1,4 +1,5 @@ -import unittest, json, six +import unittest, json +from io import BytesIO from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \ JsonRpcError, jsonrpc_errors @@ -18,7 +19,7 @@ class urllib_mock(object): def urlopen(self, url, request): self.url = url self.request = request - return six.BytesIO(self.response) + return BytesIO(self.response) class TestTarget(object): diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index e978f8970..741948359 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -1,5 +1,10 @@ import struct -import six + +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from gzip import GzipFile def gunzip(data): @@ -7,7 +12,7 @@ def gunzip(data): This is resilient to CRC checksum errors. """ - f = GzipFile(fileobj=six.BytesIO(data)) + f = GzipFile(fileobj=BytesIO(data)) output = b'' chunk = b'.' while chunk: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 983a3c8d7..3f11d4ed2 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,5 +1,10 @@ import re, csv, six +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + from scrapy.http import TextResponse, Response from scrapy.selector import Selector from scrapy import log @@ -47,7 +52,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None): def _getrow(csv_r): return [str_to_unicode(field, encoding) for field in next(csv_r)] - lines = six.BytesIO(_body_or_str(obj, unicode=False)) + lines = BytesIO(_body_or_str(obj, unicode=False)) if delimiter: csv_r = csv.reader(lines, delimiter=delimiter) else: