From e4bfb27f8351a9b59a36c441c6af87c767b04180 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Sun, 4 Jan 2009 10:50:14 +0000 Subject: [PATCH] Removed already-deprecated response decompression tool --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40639 --- scrapy/trunk/scrapy/utils/decompressor.py | 72 ----------------------- 1 file changed, 72 deletions(-) delete mode 100644 scrapy/trunk/scrapy/utils/decompressor.py diff --git a/scrapy/trunk/scrapy/utils/decompressor.py b/scrapy/trunk/scrapy/utils/decompressor.py deleted file mode 100644 index 67849ef48..000000000 --- a/scrapy/trunk/scrapy/utils/decompressor.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -Utility for autodetecting and decompressing responses -""" - -import zipfile -import tarfile -import gzip -import bz2 -from scrapy.http import ResponseBody -try: - from cStringIO import StringIO -except: - from StringIO import StringIO - -class Decompressor(object): - class ArchiveIsEmpty(Exception): - pass - - def __init__(self): - self.decompressors = {'tar': self.is_tar, 'zip': self.is_zip, - 'gz': self.is_gzip, 'bz2': self.is_bzip2} - - def is_tar(self, response): - try: - tar_file = tarfile.open(name='tar.tmp', fileobj=self.archive) - except tarfile.ReadError: - return False - if tar_file.members: - return response.replace(body=ResponseBody(tar_file.extractfile(tar_file.members[0]).read())) - else: - raise self.ArchiveIsEmpty - - def is_zip(self, response): - try: - zip_file = zipfile.ZipFile(self.archive) - except zipfile.BadZipfile: - return False - namelist = zip_file.namelist() - if namelist: - return response.replace(body=ResponseBody(zip_file.read(namelist[0]))) - else: - raise self.ArchiveIsEmpty - - def is_gzip(self, response): - try: - gzip_file = gzip.GzipFile(fileobj=self.archive) - decompressed_body = gzip_file.read() - except IOError: - return False - return response.replace(body=decompressed_body) - - def is_bzip2(self, response): - try: - decompressed_body = bz2.decompress(self.body) - except IOError: - return False - return response.replace(body=ResponseBody(decompressed_body)) - - def extract_winfo(self, response): - self.body = response.body.to_string() - self.archive = StringIO() - self.archive.write(self.body) - - for decompressor in self.decompressors.keys(): - self.archive.seek(0) - new_response = self.decompressors[decompressor](response) - if new_response: - return new_response, decompressor - return response, '' - - def extract(self, response): - return self.extract_winfo(response)[0]