import unittest from os.path import join from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, Headers from tests import tests_datadir SAMPLEDIR = join(tests_datadir, 'compressed') class GunzipTest(unittest.TestCase): def test_gunzip_basic(self): with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f: text = gunzip(f.read()) self.assertEqual(len(text), 9950) def test_gunzip_truncated(self): with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f: text = gunzip(f.read()) assert text.endswith(b'') def test_is_gzipped(self): hdrs = Headers({"Content-Type": "application/x-gzip"}) r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1)) r2 = Response("http://www.example.com") self.assertTrue(not is_gzipped(r2)) hdrs = Headers({"Content-Type": "application/javascript"}) r3 = Response("http://www.example.com", headers=hdrs) self.assertTrue(not is_gzipped(r3)) hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"}) r1 = Response("http://www.example.com", headers=hdrs) self.assertTrue(is_gzipped(r1))