From 1d5967cee6c8ed9d42db69006f75f27c1ebe27bc Mon Sep 17 00:00:00 2001
From: Artem Bogomyagkov <artem@scrapinghub.com>
Date: Wed, 10 Oct 2012 12:21:59 +0300
Subject: [PATCH] httpcompression middleware improvement

---
 .../compressed/truncated-crc-error-short.gz      | Bin 0 -> 1930 bytes
 scrapy/tests/test_utils_gz.py                    |   4 ++++
 scrapy/utils/gz.py                               |   4 +++-
 3 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz

diff --git a/scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz b/scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz
new file mode 100644
index 0000000000000000000000000000000000000000..1e0f9ddd081031f5aacc35476facb222b1940f66
GIT binary patch
literal 1930
zcmV;52X*)#iwFo$sdZ2Q19WA0bS`LgZEOH_SIe&4wi4XuE9jLsGh<|Z4t(Ln0URKX
z10-2wNlC2{Jrt?%F|u~~i2H^4l2kRNITts{!2w1OlKrZxZau$x|LNObe*5V=GnLuD
z{pz{>81p=d9rWVO<Eo3p#tZ&EPhY?NAZfP2d=!6o-itY#kFnd%?_GbxuCL#Ib*vrS
z{n&@Q<>lI#cX#W&3s*C_Ig1}n98H<T%+3!dlj}T7I`=Vpvx>!wgtxYs5Lc7LU4sn9
z1@6dVye+QOLGsDipwE{ii%obOgBfGelNaf$cQ*OW*uJ<sp4619E7{~Jrh?}snoe*=
zalKn?5qR>Gq%xiKfKIN-&3xE-my0C$5dDBV+u!o+xNfood@S&Bj(5DDTs8|1w;l2X
zPp(2EJZ<7ud$lgW>k+4!K8D`cp(Q2rwj|fp64PeZm~I-$yUBL1v+;6;8?~5RPy(1M
zIEXnZ1|bj9-%e(<xga-r!gCl+IJk=*;dYyPfa7GwnkPg?qs*=KbHpweddV4yKvG<|
z#~Y5=cM%nNme6uN*|fwMMADx8F0On_gF`eq&M1#71<`+w-BFAA35F0dtey-Wv0CnN
z7i2|6>_VTg*d=qf$;Fxpg(Zf(8v-20ngU9n<YGP{hXLLpRh2~%fGDL{!_a&OV8vdU
zW|w*|@D_Pntt)Cq3!f#SW=~ogKBb9sUaTT(aW^?@@TVi{J!Eg0t#<uctcRya+iFWc
z9krCZpolrfvKayc87kP2>2Hb7%WyG2auMS7$UFguZG;GzhI+49YYi~zuoL!453@S)
zM|6*OHqAIInz<(YpHNlf=Jv7|y<MbK_1O$=0L)OLF~yloyqKTi41LQ!t}Ub-z0Bc>
zm1Q{aKJBPO_LZ56)+xo{dL(r-T>@Z&=`2zBh|t*8ZW^>3^%2UMh>lSJTWUbFw}b4C
zbHIExIjV_0>VJPPHkFHcr#_PjNZRGkKmPlXh@Ks)OBTK|?;cWs75>(A<R#ni!t|&F
z2!$-T>r|9zKxi1NH+ETIhh{gc3BfY%29rzUKX9pFqtYw@c^nRLyT=U>H1ef)MOvhN
z#e(dBi$ef;IOx;@*~J1CgoxDevJR}k*_k>tnBS{pY8~8BNYRVY1ue!EN*Cbd(_Ty_
zrnj@|&|-6Ly4nB=BC87ANI}{K4QFB0SygCH^a{D@sVXBh@?yRb-?WkjD1I)4*bBwu
zq(^oT0EKJ755PRTAuz9et7cM!!>NX#tdp_|6=aP7XN3q-nAebZ6@I8K!t~=gm%6wC
zfsW#ft{6a4+=n`M0bGLbAmszPK>2~|%GxHxhnk>@7OHB4s?gv(^k-mLfN=^J^9!qx
za}}bhdEz4u->W8H3d9s*BKoL{aBma=96td&x(JC&NVC_j1U&&*$z1-Ph#610*2=_M
zc7d*mz3O4*>g^Cz9uA@~y*|=qJ^{BjDW-<r7MzyiGC7vXjc^ACJ<dr^#-o}fAc`_1
zJKKx-AWdt+3mw)es)O7~_+npZ=+5p-*&65<kPO-^%siBab<+l^=ql4rlpG<+E36gi
zCzT5*2Vg2JzYA)fr*#AXd_ZnorY|F~hSp2!2$rjqG2)dtSSVRrYdKE_P@&jB#cLK_
zbhT*^5J2WG*<N36$ngYVqtuo#z_8&V@FyhmPG46f<;Zdpq7H`eB-LTqAa<Bik{+qN
z`59ftLh`A<@B;cVp(En>vybt{^ZKc#9`nHh;Vo@;u_yx&3H}fvWM1P0I~Iolf)#NO
zHr-xoen>871)%ykNt<@R6KHVxS|N9PlK~$xqykumjwL`SQP#K-t~Om=mJR8e_lQ&M
zx~5LEHc2?4VQLkdEYT@=L9kJwP?EosbD!(jV=%A9)TRkIKGeQ6J{<l$TtShY-(Eko
z{CgE~J^GudQ^dD1e6A^`SLWAcSC6<z*xP`j4XR{43?<G$kWDbr?|L?(3+_UdrvznG
z=+&j=0U2QHTy+~}NQ{)speA=kLfb|`2yu2j*lctByDYudd2&(5+lUGYY1KF=H>@zM
zLc`9Y7`B)`bYz8=K=QOb7mD>0HCyAd9t|`rKX@o0jG%P@X_!UMe3>@IuF(mVM9|aG
z^=oK#MsK-;84Xeo^LKZRi6a)cWHC4)*1h#W`LT!%?`X9wJZ+ybYV0HMRC5PUULQ^R
ziJ}4aQSeeHZ4!sQxsIc66AOD!oA45axRnQVNo~ACXZF~jdWiZEAEMI?<t?K@Xm?@n
zg*yR*JOFyMIQ6~x{e$fyTaLqYVtn=9cH2J`46PmT%xxk`vA?yEIeYXFNf2>Rca4-=
z{=Cxr_xvuQIs()-WI{Stb&ykU!IwhjP{|9w{l|R3<hZXXmgCW5=J+4dXwEm~4Lwmx
zRL=Jh$0HXpv>t{a!vBF`bpyJ$&LCjD$5#wXs%pQF>(rFAOqN5mDXD^y7!TTHbmCV9
z_+dG!x}^An%GsG!Z%D?}(Y{qS@H4l3?-J4E0!}75-<-mmvn7U2sP%Sv`3!vX_zrx2
Qvwt8yziFSC_<H1L=z&10tN;K2

literal 0
HcmV?d00001

diff --git a/scrapy/tests/test_utils_gz.py b/scrapy/tests/test_utils_gz.py
index eb2acdd13..c7107f6a8 100644
--- a/scrapy/tests/test_utils_gz.py
+++ b/scrapy/tests/test_utils_gz.py
@@ -22,3 +22,7 @@ class GzTest(unittest.TestCase):
         with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f:
             self.assertRaises(IOError, gunzip, f.read())
 
+    def test_gunzip_truncated_short(self):
+        with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
+            text = gunzip(f.read())
+            assert text.endswith('</html>')
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 57bcb2d60..7fe82ed39 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -17,7 +17,9 @@ def gunzip(data):
         except (IOError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
-            if output:
+            # some pages are quite small so output is '' and f.extrabuf
+            # contains the whole page content
+            if output or f.extrabuf:
                 output += f.extrabuf
                 break
             else: