From 7643f14c88d51294e1eee99797dff9f68cc63a30 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Mon, 6 Jun 2011 18:25:14 -0300
Subject: [PATCH] fixed bug handling truncated gzipped responses. closes #319

---
 .../downloadermiddleware/httpcompression.py   |   7 +++--
 .../compressed/truncated-crc-error.gz         | Bin 0 -> 5766 bytes
 scrapy/tests/test_utils_gz.py                 |  26 ++++++++++++++++++
 scrapy/utils/gz.py                            |  23 ++++++++++++++++
 4 files changed, 53 insertions(+), 3 deletions(-)
 create mode 100644 scrapy/tests/sample_data/compressed/truncated-crc-error.gz
 create mode 100644 scrapy/tests/test_utils_gz.py
 create mode 100644 scrapy/utils/gz.py

diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py
index 26069d437..fd103775b 100644
--- a/scrapy/contrib/downloadermiddleware/httpcompression.py
+++ b/scrapy/contrib/downloadermiddleware/httpcompression.py
@@ -1,7 +1,6 @@
 import zlib
-from gzip import GzipFile
-from cStringIO import StringIO
 
+from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse
 from scrapy.core.downloader.responsetypes import responsetypes
 
@@ -18,6 +17,8 @@ class HttpCompressionMiddleware(object):
             content_encoding = response.headers.getlist('Content-Encoding')
             if content_encoding:
                 encoding = content_encoding.pop()
+                with open('body', 'w') as f:
+                    f.write(response.body)
                 decoded_body = self._decode(response.body, encoding.lower())
                 respcls = responsetypes.from_args(headers=response.headers, \
                     url=response.url)
@@ -34,7 +35,7 @@ class HttpCompressionMiddleware(object):
 
     def _decode(self, body, encoding):
         if encoding == 'gzip':
-            body = GzipFile(fileobj=StringIO(body)).read()
+            body = gunzip(body)
 
         if encoding == 'deflate':
             try:
diff --git a/scrapy/tests/sample_data/compressed/truncated-crc-error.gz b/scrapy/tests/sample_data/compressed/truncated-crc-error.gz
new file mode 100644
index 0000000000000000000000000000000000000000..2eeb4da1640a0df90bd652e17af24284bdbbde35
GIT binary patch
literal 5766
zcmYM2do&aN|Hr$RPoW}`YF+6<a%<(bZc>Voq>(9^g=Qg!-HM_}t0u#+LL^B{E}Ly7
zk?Ud~iP`2Zx6Ng?*~Pwn&hL-k>yP(&o%i#7&g-1l>%7kMeI9`1b<6&XRm-LsYj147
zkhb5*4AG3nJ1;nQ_uqT7?*0v(eZQZd+&K01aogz2sU1O1dk+>Lde98J3iA!rJJ~v&
zT`T3IMU4MDXtvp|#!`VAbUiR&`5j5Sb8l15?)&e506;hl0JP_2BWsdl2Iear%)p!&
z`T-8f&prkh^%7Tsl?qX!>^_D<+{O`0mn3AfoI?6gw%8@xl$>k&9(I+_K+6a7#Cb8?
zk2`q`bY!G8lLq`akxWNIdDykr=+O{7FO&gnq((_P++W&_?i83d!{#BY+J)Z|wMQa@
zmC-qHYF;C(3`1_8R4S<ip(}Y*I4PmElFe0ecki(gIKW!@)sD?2bV_1h;U(^5R){D(
zp6+zPnLQ+kL@UtWV`8H+9b`<*Lood{+Z9l2)+ZWifrKrorV*;w>*osb9X6$<jblCG
zvGzrLgy@+l1%>22xj&S^xdSmx4!VlV@3)`m>RHJ3Za?yicp=I@=jzac4~}lyYXOp{
z5CMu%sB5ufPNk>Zxwn84F?7!sYsuzZtTWFZb|LGvRCtd>9&<K^Kt>Un?%bdZYfZVU
zeu#Y0(3l|l=TBpDZCfX-a#t-p#<C=N%cvsnp0_C%(}R<^7t%emf#MkA)qi2~@pf0|
z>};=vxIaW-P#Sa$k{!sQ1<!PQnzwlP=Ox>fv@gGx*^RK=tZQH$4<jEbr+f_F))ek?
z|MpgP5jXgRv(Gkx7fT=5ZWQ^dtszvtYN(>9FfFGC3>P)%dq%pC_PpQaP^_oy;gzSS
zkJr*|`2bNyjjwLYEExDzthyJKp)o~xwo~v*u*Y`-c;<R}a{&^mT{BM@zuoV<cjdUK
z(z>`{a<xlyc9_cw6eE%ebu~}+L3R4wAz$DS@Y^jiy1AMzoO`YT&=M{|^wfBSwiBnV
zts(d|qaZ;pAh8mNyu+!C1VlX}f#jhMyM2j;rHgceaJ$yBpfShz2S*++2Y>nH1IXsc
z$Z$t!y93R#J<w8Bo&Kr@*Sl77F%FE=_*}jJZ8a5{aD}%UZ(qmIFS5&wPMy<`G4p9?
z{dLdCgZ?#8dt=1fZADw}sQj<$z@k+9pPcT1f}BweQm4^x#P=U2oIsLUf@o?A(|TX!
zI%BE(ykf6`N!(D`mb##;UXw;<NVD)W&mXp1#YxwGK6M?ok(3YM278(u>121JZ2~dZ
zObW;0wh1)g<Im(3pBlUDLdd~2F57VAPt1~QhUZe4OY3e`Ke{dW=ZjNeemB>pPStk4
zvaxt~9{=%(OSmeh;-5yv!gv(gn^ARJyYqb$7x;Zo>96TuPCc{|DAC$1=msFA(|2=M
z_RgD=$<yVuck9N2AeRJMVTX1ejrD9V=$gp<t<IaRtxhM2@pCy2?qz>oYt$LW?0)d1
zv$L5aoY^`%3?pi@)J?`@fzk45>#xyii?75>9Rv>h>x0`I=~lU@a{p`57eSxAy7li?
zIo+kvJbHp;@zee%S_JMsr%&|io$sTWuWV2Iz9%H&^7zSk-5B}BhdYjaL5Z%MQ;P;a
z$N%<e-qKN3O$FO#b?e+Mvt<X*1!DIOqxQ3EfSQE`4wR6?1aRo?P1#3gY5}E!`Z4#N
z(&JVQ+)Ydg|IggKY0RQZCcA2f@mXS6i<rg&ejHp;+V|sA>!@Tk%zkO$`r0L8-Gm`y
zt(!{9`hnZ@A`7o<W#<bN4l`f<hPr4ONUUf`K-szwZBz2=;>N|H(q~mt6o~gX_-dH~
z6`OL9Nnv6$e1}PnxVM^ey^TBk0{$(Tb!zaKbLv%Aw9a<_YK$9hwE8#s&eP>?T!QTx
zSvspVGemYg$fq=7_kQeorarb}!Mv_&w4?C`j_Uz&vD{?|?eg?-DYj7AUq(brLpiB{
zLc%ceNgjxKX1lCx4W4W8*7PIS|IqGT3t~=NV`9|#6i{w*ZP$Pm_@f{|-|YOO>IYcl
zm?AOgOviN>P3RRW=DLYt(k;rcu58ptUpEAJsn#!iZF7T*aEP9@{LOgw6>7+W&L)$P
z`fTfB7~dgJ+rjzZMv51e?cnXHsd2d>;^gvC(X6|7Yg}5YR<zyWBeU_m)gu`(=HwCG
z>h;3{wN%|i<%pQL2<7ONWvc+s;bjHqP^GSZ4h;E(_&eE5!je5nL{fY)@Y#DJV<x&*
zhJ`NuOB9R^%LbDvvLZHqkPd#vW`3jiWnsCBC9!N!JkmlOl?@MyW#e?gn3$_SjiL$^
z-zTJy+QWX>r8{GVxIyU&2V7<B;;4|UZw!`znZ0jN@oXH0i1bqqyGqckVB7>9w3sI6
z;5156GGUh&XaUVe<pi`5&2pHGXoj+B0nBh<E)9;4(@7Nkn0#uupGcxmpaFs<*F_Hy
zgNnf+uc{na(qh0RiNvf>3iX8xuw|Bu60W3Bf}4w_0a+w03com4C{W#yFH#qJ#>_GW
zV^Z8~JgF4^Up+j0Rw>1O&crT#RJLmiSs7S`ZLwgCUu=X|4Rggj?Q$ghDv`q6lMWsw
zQuv^L%6B3q2eUM_mIIhFfhSY0vzZDa$YrP+EMn#K9UdbRjlo6~d|}oR{NUii7(276
zDGVJY7lxNdM~;7sWuoLy4AytZhleNnN2tlv`CFMJGn`z;(-&YDB!x_@<gWwc2c<T)
z(UEs&ruwI*wV5g%6qv=3CPVjlMCfPT^*X*n{YWFJTiIA#Tl8ze3IG=cvKptT$wMt8
zEpRVePCRdnqRyR=vX-z1YUbu{#<<~d;VS<nUX`&QQzhh%8Lt(n7`TomFgDCC=JHbg
zNIN)dX}Cos>{F<UCwbl9!MG|})Ay;ud6hsxr>OaUDu+_7BLXHJwLk>b$QcwSP7+&&
z2C2s>I7z%<tnim#+Yu6V-b|WoMoC2q^+DzkY|pQ5TR3O|bJ>rP#rnwY{yTc4y1t1>
z6rj~XDjhoOsLuy2E?dZs5(+57_RvjOgY4Q<+Y1mi)OmUn>+}n=^S0NA7e1a}i=S7t
zuy`Wtt^nQR0R+LRdO8ET$tYvEdw8!#97m(*V^wNrEW8|x8fhV1uCud!sX_f!zrwNC
z^l)0t^#16$H80ow!5CJ!!Lw#NKaQieMh>hi_tcMx@FJ&pT`Zr2^t6amI*lXCYG140
zc6VhG?;5^*Y8S$@8gfG2y0v3#XJz2_y=S;TW68>t2S9(M?}crV9K(5Do=ej&ZKHpB
z9`ZDKjX^M0C1%ikDi{)Mbm5!JzNg4FoAOhSxz#fd40h?illUrnBZW=H)>hePy^yPR
ztUKg#<TXr`RS@8KW&ho-#S`^&Mi1@5wex*x&C8!u8c)v|S*$+v4g%J5H1s&NstHhf
zxd=>hwJm<KX9sfUi?WU1MBpEL>zvo@J{dDF@5m2MME5P5x>`qXJ(5_D@eThrpx5`_
zWCPLdo4EEJc{k&xUs&_cS{t~_Of>}7l85Ao0^DZ~U-`AhhV&gVOM3w^an4**o}R(l
zQX}f50D$A97QnHDok`T(j6H~uunNF|m*52v;Mg;*?}4tBUo!obZ{9Hq1o$7fW3CnT
zh-%j9*~<5+CRT33U{JnH(g!W<_O6vXtT)C0m3(R7pb_s_#;z*+lJ4ZDWsb<?8pVBl
z#G%A}^r8K$1lb-Y^pEq109zY3^om$%T$%-(Tk$j7w92g9V*U1FM~B1Zbc#?2>SNG<
zd@87>ZOnE>M0GXQ=>uj>q;I+(x_(x1Cd+?tr-RK_U{FXD95p69{NWMKc=ZR}L)HvD
za*W4UWxsB;pPuFBj^^4QL-;-~s^J@XRb}{@2lE^Ne6FGbl!Io>9-q^aRXTirar{Y5
z^*L*DYG>#KjKu%%r+VJkV)jRlK5<s0k}r~xWBY5M6l}#u5_#Lkf--1jjJYx|k8REv
zC#Cbn5t+~(&dO|qk&~H(XA)td0Ae<O4+%%1JR+doOMm`&M!Mwmhn;w%JS9iW>Ff?e
z5duok9L1E|j_hHWLs1c2V(-<p#k6iE+}qYzU%V}ON;YU}cSW8(r<)owP@7<|9@VJv
zP#S<tG1N<)OCQnAPi>Rs9``yXi*s<HG&kNg%o4C}UP6QZZNjb|_xHyp?w`_&peyd%
zE%6fVzE-7kfoB)Zq@A~%bzmZ3NwmlDp(hWuyc{6~#Wu)KNvUt!4pAz5?)hFGOuBrN
z`gvbTPy#*AnVNRu1{{eY8wm2lnTr}M>6z@A?)QD2Y|k2=e*=J~q27H#%&Bb`Y~D|W
zxjOJ2GfWS7oN4*JkIuDv!4=!O2%qX%&fR<8NvJ7}xSZcIsuxiUX~O?BLEU4T*YQbj
zc0k>(uJPaiwF0Mc2ShiS0;4z855H>A2tZ$p2gt69Q3FW8!lQKR(qY!5B#IJ@U|v%#
zt$F<0Dd}QW6>`EADj3%9De#J{+zbWG{BAXbeJ}HbxSuxeyas<aU;k9nKlQ@w_toUx
zPR=ZTgGI|vlc-<xTdUFqI;`K#>lcBm6Y||R1~}Wpf_qU0Xe{E=T|42+A}PY`Kw@Lw
zGt)xo&o)vEHy^rn*iG;I%H*fbsMQM13$HjAe<!?2O(MVm*MZ-1Z?xa%o~#@>mPb1%
z8X&v~H?)g0B<@-|PcuWD4Do2b^@seGjaAv3B%QK7^<uMVvK*1{mh^8@jC(z^l|=o(
zPY5(eF6cI!9-lAXV$yrD{LqNdR41-LL_W<uMowL9XgA$=?L+z3+&^=JR6^9S$nzL^
zR-pUlQ&GbQ@cY;`L@>f?DHU8<ngp@MSI}co?*KF11n89(<=vb!2vF|Z9)Ze@JK%*l
z-|`4n89e7SN;+dyPp}W2%xj!Bc}sY6-a+x_!6Z@dtl96u!Rhxnaf038IlZMzNHk^h
z5i9G{YqAjstio(#&j<-ql6N-OeSvqnY+ei1!Y@Cr;i@jp{gJ9w@UDyA?wHdnx$@}q
zdTki2om^%tMTvG`FpW{W0-`T>DxxYKN}q2VTlVm2*mC%z>sHM=XO;h~!0MOS6%M|Q
z8DC$U+Upq?)o`Zmhi&eKjLrV%fvZ<yKgB<Y<!3NCdkHIUQkSUC9}=~ZE+#iBGu&<u
zY?@dNy|HJt744$kR}6*(`Kh>44HFV?rTO{<Z+#g^yr3xvZSuR^8@j(5e^0#nL55&w
zJvovuzEK+k+A3&+x2#JQF#)N-`AF5*cXq|eJ0HVEYX_}64efW83Uq$ZK$id1wUxtS
z<s>_=(U&RZRJuB4R<$H7)DFF&K--S%TP5V7Z6DlKXANSAP<mDIaZ$tFv=bro0HX%r
zK~lD_mghl_S@q_pfT_NRvpN2?QnlVw6tYg1<Bt)kO}C184`-SFwd%jO|E1XPGTiOa
zJwmp&;ev|nroQ5?7OPF`nRuNBZQY@@YUEcLhkVq*?WuOc(Sl1Fu=Dc<J#D)9vB#Se
z6L$GxP_7pz6YUA=NcYY|1scZ8j++vM(NKTFqFdtPT~2+U9h0Kjhb+&g+cs==)Ldel
z&H*zzd_&FwdpjsP!`h+c@8}J=!dRmP^^WHGej_d|<j$n<+>kutjEvkm_0gP4H=Leg
zYt8{7eZIHCQ1(QHT5lLxUPEsf6^2^TK)@e@030f1k;l7@2kuf+A1htbqPwpR@=kdu
zBNCM^=0!)QHqGTcP~o2x)HX4N1`1kL2XpDDJG1G%(4d$mRj2(C54cp`W1lNjBQ?MP
ze!ywzQw^R1e-R^}G5S^m-!R5kD>jUMBX1bXG0#F9YL02^jF4MLa?DBSoEjxQk&lih
ze(tmU75=xo9TVTkcZPGyUZ4#T!~eZA8OkYpiZ(pM1{gI1jY3jRDe>+Jiw)m*wRfj=
zuUzb<tk050g}y3V-R@45Ma0jSgjas)p*~TYL2P=sHJ)<4A5JAhY|Zv6=l&IA@7$F&
zUl*@t`1Ehr+=CCj%k&iL9d{}j_)l-<|J6_YryIrpD7x0a{c|D+cwBI3bxXr{>f@iW
ztgyLTV#~jK?fz2Bjv#7JYLRsl#>o}w?u933q$Iq(6<*y{Zuf}1lpCBpPx#3rXi9J0
zFT2HJq<XC(bG=gMOiLa|oV{DIeBgI-QZci3)*AQuue+zjA2I&=+8RZ9iRi`J6>;cM
zy&xvqn%!IA%}4aocAf0z>c{X}OOr+#ojM8qve`qW?5Y?GkV9+<g2VniJw(5~jd@a4
zIv>;l#bd<pY3<G8>a5{NGS2pJ%U|jNHHE*zZZg^ZxM?l$XTSZpZBcKfcONd4skPSv
z;KX`GSRnALSYexE+ARDbCW2$!^sAY^n+Mk2LuOq&?2Q1wpFot=-nJWHwg{{Mkhs(>
zO$F`x@^Qv^Tg#Vsm<F=~<a`v}24swzp=WPttH|AQSo)W)d<gf?%k>UDS-5_N@hNDu
ztJx_|y>rUh2RZd--dzf|ghULMAqGBA)2h=^u%2VrN4;D!s`80Ni}c_5fqkK!NBoE@
z)JZ>qx88udpCT(aIWGMWg}%#MWaR0K7l!EKi&^jW;tr9j<K~R-`*3;(>L`y++2;|%
z#DT6Cws=q8;XookUou{FEW3m27RT_df@}+WnlD!d2l>3!_jJ}wwwMme6<F^eGzK1X
z%!_uu;5Kyo#mjmWN+^K6`kk%x(Kb39nDJGup#i3S__&T8=s3yvYqEin&Z5F%T4qUM
z|6+v=m@xjf;@W9L5pNrpp+KP|OruKdLA%oCa#}AGxF+Tkga&<3+2#Elm*x_2;GsBc
zqc=?dgpLC;$YQQyUw)?EbY|+g)vx&NZ)`ffdaPYICMt}5MnHEJ<ykh5PN#3}Z#T`y
z#&7;K)q91r=kVIrp!<)iol-k|tjC4zS_J0<;*Ji>r3_rxs`#z<9FNJ4Lf*xNTAm0Z
zcb%POJIGb!+S(4C*~EXLTincGk?xR@%2@>VX8g$Hf$i!=eA8>TaTP#R6E5xEdi2_t
zXc{c1i*SB~4cg|P<P-6UrV)SDsJD2)5z#j}W@H%F(s8ZHru2Q(Z_QK5@IHS<P3yys
zQy>R6)!_6-%8pJ)iR-1FWv>#@5KovnW9MdJWX&n9mbyLeV?Erq4Mx3T#6!Ma0k0iw
zVxIskQrlXkvQUWf&dsX#$t@cKVh*zM%`z^5x^G(_J#P?WS~^Y=C*N6!+$1jpD2|?e
zY_4zRWgI}MU#X%!juwQ<EvEB+!e8IHT;g*XgX(FaRNi?79!6E#0^$aT=x56BUNTag
zwX>1e&;k=9cU9!yos;hFy1(E%iQiCE5eoWdVL}V=F=anD!E3HBf8gfFJr`O|aQ6r!
z5_0o^QGXIaO0IWHN?Q20p9)L-Uh}}1r~!kN|7WEQKr!2wZJVCjGdGRin%Z_MYnRl}
zSohgQJrhK!SAY&z>5J|jX?T8;hL+BX<P2hABk~z5QBqn;o}CSLOga|tY5<H;*pvo2
zw+H$>&}9U!^9#5*_jZy;I9G%zF8bVFh0?n939rUHxztmYTu?GGNrlaH>hZ^!?j{#F
l?%QV8p1}WQyiykaVQTz_M7(|J#P?0AtiRwvnN^iM_dk~|eH;J)

literal 0
HcmV?d00001

diff --git a/scrapy/tests/test_utils_gz.py b/scrapy/tests/test_utils_gz.py
new file mode 100644
index 000000000..3dfd1a2ce
--- /dev/null
+++ b/scrapy/tests/test_utils_gz.py
@@ -0,0 +1,26 @@
+from __future__ import with_statement
+
+import unittest
+from os.path import join
+
+from scrapy.tests import tests_datadir
+from scrapy.utils.gz import gunzip
+
+SAMPLEDIR = join(tests_datadir, 'compressed')
+
+class GzTest(unittest.TestCase):
+
+    def test_gunzip_basic(self):
+        with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:
+            text = gunzip(f.read())
+            self.assertEqual(len(text), 9950)
+
+    def test_gunzip_truncated(self):
+        with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:
+            text = gunzip(f.read())
+            assert text.endswith('</html')
+
+    def test_gunzip_no_gzip_file_raises(self):
+        with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f:
+            self.assertRaises(IOError, gunzip, f.read())
+
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
new file mode 100644
index 000000000..2dc9cd3d2
--- /dev/null
+++ b/scrapy/utils/gz.py
@@ -0,0 +1,23 @@
+from cStringIO import StringIO
+from gzip import GzipFile
+
+def gunzip(data):
+    """Gunzip the given data and return as much data as possible.
+
+    This is resilient to CRC checksum errors.
+    """
+    f = GzipFile(fileobj=StringIO(data))
+    output = ''
+    chunk = '.'
+    while chunk:
+        try:
+            chunk = f.read(8196)
+            output += chunk
+        except IOError:
+            # complete only if there is some data, otherwise re-raise
+            if output:
+                output += f.extrabuf
+                break
+            else:
+                raise
+    return output