From 892dd9da57c5cf005670743c8abfdfffc7027acc Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:00:58 +0100 Subject: [PATCH 1/7] Adding support for zstd in HttpCompressionMiddleware --- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..b1abb8b5c 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -14,6 +14,12 @@ try: except ImportError: pass +try: + import zstd + ACCEPTED_ENCODINGS.append(b'zstd') +except ImportError: + pass + class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be @@ -67,4 +73,6 @@ class HttpCompressionMiddleware: body = zlib.decompress(body, -15) if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) + if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: + body = zstd.decompress(body) return body From c6c3f2ce661ef9296302b9d489d5a2cb2e49f481 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 21:10:40 +0100 Subject: [PATCH 2/7] Updating the doc entry for the HTTP compress downloader middleware on zstd --- docs/topics/downloader-middleware.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 06e614941..9645ed5fd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -684,11 +684,14 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ responses, - provided `brotlipy`_ is installed. + This middleware also supports decoding `brotli-compressed`_ as well as + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ +.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt +.. _zstd: https://pypi.org/project/zstd/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From da3171d4f71d50a15b08b76b2b1e06bddfa56694 Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:18:58 +0100 Subject: [PATCH 3/7] Using the `zstandard` package than `zstd` for supporting frames both with and without the content size info See also: https://github.com/sergey-dryabzhinsky/python-zstd/issues/53 --- docs/topics/downloader-middleware.rst | 4 ++-- scrapy/downloadermiddlewares/httpcompression.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 9645ed5fd..7c63a623d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -685,13 +685,13 @@ HttpCompressionMiddleware sent/received from web sites. This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstd`_ is + `zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is installed, respectively. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt .. _brotlipy: https://pypi.org/project/brotlipy/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstd: https://pypi.org/project/zstd/ +.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b1abb8b5c..56421a6ba 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,4 +1,5 @@ import zlib +import io from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse @@ -15,7 +16,7 @@ except ImportError: pass try: - import zstd + import zstandard ACCEPTED_ENCODINGS.append(b'zstd') except ImportError: pass @@ -74,5 +75,8 @@ class HttpCompressionMiddleware: if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS: - body = zstd.decompress(body) + # Using its streaming API since its simple API could handle only cases + # where there is content size data embedded in the frame + reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body)) + body = reader.read() return body From 50e1f35d1fcaeb2e0a0c4fb29894cb7c686a33cd Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Mon, 5 Oct 2020 23:43:12 +0100 Subject: [PATCH 4/7] Adding test cases for the zstd content encoding --- tests/requirements-py3.txt | 3 +- .../html-zstd-static-content-size.bin | Bin 0 -> 8066 bytes .../html-zstd-static-no-content-size.bin | Bin 0 -> 8063 bytes .../html-zstd-streaming-no-content-size.bin | Bin 0 -> 8047 bytes ...st_downloadermiddleware_httpcompression.py | 26 ++++++++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/sample_data/compressed/html-zstd-static-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-static-no-content-size.bin create mode 100644 tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2247ed917..2eed2f5da 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -16,6 +16,7 @@ uvloop; platform_system != "Windows" # optional for shell wrapper tests bpython -brotlipy +brotlipy # optional for HTTP compress downloader middleware tests +zstandard # optional for HTTP compress downloader middleware tests ipython pywin32; sys_platform == "win32" diff --git a/tests/sample_data/compressed/html-zstd-static-content-size.bin b/tests/sample_data/compressed/html-zstd-static-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..b5c2038e893481e2835ac74bc790e3718986e426 GIT binary patch literal 8066 zcmV-|AAR5`wJ-goRU`obmHPmitfU$&;EqO3lFJAV16>@^C8(hY1ZsEGeDXVhj51gD z{wbwTMW0fDVIjsC8zUwJLIPd^Tmj940%8FN4lqZfU`(TE6bp>O3tV-7e>YaR^FV=)2Knk9h* zPD2Bxu`I2bEJ}oPC|O!VVVHKB!!*Tl8rtL0DmWw>I4vT=w8gSua2St>L(&)r1jK1r zB93OkEFwfW4Gr3a353%c%i_@@6d(g3YGzZS2 zoTW`Hi$=w=8$L-PHPy)01wDyvS31MEJ`~NhjNz41g9-ZJB>s+5DPeP;2ej=V*+6! z2AsyCEXzc~31Ax}lf`M!Fs)Hq1*RRwGC2*?lnG^m!iYdlH$+BRYv$Pg*Bn!uBmIj$fL>Nb+ zI3N}nq$wB$#lixsfDwsMEIcR{9u$&CQHqNO3cLt#DP1=;z9HvN0E*SB0B*i|V@? zo!0dgG3lazM?J5RBBIe@s@IB`{`D4B*O(MBjS}6Wt}`j>9cd;_G)lU*(nS1E>Agxb zjnrOTx<i{{ zuB!54rdz~ydTv6L_*KKIR4?Lcu5WP{s>|c5E+#Mj_g#pa>{ZT1y12Y}n{th}_Fg9A z1<pTU+@2?hd|hVob(Mx!|A9L6+?2gL$o znujl=Vp?bP`Io;tI|2zX28DP;Bu<1Tv0&gpczD1Z49GyhvY;TT5Mhqv;Q{e@YIOW5 zqWax`XR5Q;t%z4Y--^b(_2;8+t;tDU>hpQ0m`+OFBE|bpS6Zlk`l0deZdK^3BEqA> zRR2pCDs;Ng>RwcLUCNkuG%jD}5BD@tF&)x8I_jk&p7V)G(`gZ{6xZdtwNm_#i_Tl2 zqB?K&qnD;qh`TiXT=d>C?`4E=@0jSv^u2Ns{X||@`%Yx~m|nN3bgg-Nx_tMh zb2;^SUoY!f553Y~t6QVm_vRBZ^WMz=)_7=tdGooqRzH&-Za(+c=|tQ_UuU{hqXP@< z0cji$4~m6_Csd@4D6L2Ibahm`g!G*LBSHufrT@5&`E=gt9~CLC;}Jc7L|=%~6FWqh z)KQ{@tS;?4N|cF5C{yuIl!%f;qC}+9QL>bGMwkhqyiClabV{lyoz%Zjs#H$uh%&8( zGE+h*FQsBA^U|pZX{r@dh?4q3dQWF3rYIBo$(bpW`jX0&^eGck=d==2g+9fQD#cPN zrs*jaN-9yMJ4&e7Vo6!qdOH>KdRm#P*!oJDnd^{Z>vwA1tum=Ey;6m#)M8huSXwL< zyYE+pnzz`xLRIF)^nSQ}mAF#nVoIqle<}8*>gtp&UCI#3`$D>wuEmgpUg~u*WtaXF zV@FG=YEsvw?77xb#?qIJe2AxK>MKiD`*%9?=$NK5X1y|B^~%4xGPYPE{XHF$5%uVv zh$RPg#J-Nu4u_8E88LN0hKV6`NVMXQA;#D(2W5Cv4reS8VvI4C3^^pk7z44y7-P{K z3Za>!Q7F?WBrt{yiq@z^(6q(i%pd>(0YCr&0Mioy02mAmh>Sx5DrwgN6o3I56b29! zEDRI~0t7&C;P4u`C=>t&LSZl%C>~Ta1x^BJ0QDms#I{8t+xDvw^)H%OmM^UtD6u=U zJi2g;L;Z>qIeYC%x77AvCUM0~2dZsvqsQWjcw9QI^3oQ^-Ny=pBn5wXbuZyGOE-C_ zU+(88f@qO{Z!@13Ld9B{bEg_PV51mS_}EE@qw|Xtd&KjUfOP~dg4w^u)KKYBv)CAw zot38EujidM6)6_Q0emIE5v2Dd^J_S#jAd|478Ri)!r?XqW)yq}yGjGyFZpbke@%9A z_^u{F`#3YR{645{L-~!AtK$yGb6vk!62X2+S8;$e6dz#?0lkbchOLSGA;3MZNDi7_ zDZ`<`7%Pq~Ov@c1`J>Q+#XEwZ=En{ju0tq)SgwM`F;(vui(^FjAxj@&3Tt+_cON8; zz{D;>Q?B)}jJ^@EIH)ri__uIYjQ$Y@q47ge6H$0uQ(bqJX;zw;fYKs74qQlWnM$zn zt42$TaFlT17RDKQ`q8l82lUt`FP*I7d0CAn3p5-cre}g%ko2+aBjS01-TT!%Qrs`2 zL~I4ZXS$tk!Q*FY&%tSqr}IP3GN6x2d0K&-ZUNiTB0Ivy7Z|@j7m;$}>sJ?+$a1=&4s!@7FS6X}iQx6KvgDT?Bz{7oilbL9z-4ct_rAsPzI!Y^_3E}j__m*@@3lVL2ujan z-u(=F9dJ)WDyvlfO1+zK;~Mmg=xq%#`vU6msw@-M!y$M-#>{el>f_pp|83onyRevw z@EiId#6snzKtlyy;h30OWXlx=z~ z8}Cu#3gc># zIsI;^khN5tfp}$v>?3qU-7kD0p8a@sT=af|0#_@H>GQR&&9K?&p^xx*zgWVa9Vz4P z4;Zh}^^L+z-surWhn#H`tOz}3p7HvZ9Kw{D2R?l=7)&}Gyqjjo#btC!7Syv>l%_4S zLwFzbs!lxJBv+9Xhv{~*+wa~&-|>*CZ`3;@nD5bjTf>=$P3rs}kA#>Wz7~BbuZJv8 zndrY)4{tW7znl}>9`ep3Kh#XT?jiGkI#$4%dBoD`P5%KpD0O2=r;|Mb<&a&A2?8>sh%T{-p_2KWE-H%9O1BV(V`{m17l>qV~z=?BrOh;_ok7@rd2|~ z`IqZ~j#yB*?T5UQcoG{c>rM7YkW2>8;YPtzq;4M3q*%5%@@zBzElOkE`!xUm@V@_h zF-3O)mbFwQ7h|5XRm0Ma9d?>`gzZKP8)duDY^I3gof79a|Lels+-KtuK4VlhC3Ih0 zS@X39#s>(w_aSJpT9w=&J}k{$ads~zVPbbbhdHgGLH>|G%lp-{0F#l;doyZWe3rMJ*W&^`p7joRO!v022)e(S+p z#yehuj3s;f^${>C5j_r4M@v^RvCZ_?fQoR~@4}WtcGlRGBi|pxdi$syZ}exD$01+G z!M>kq1o{kDtgmOKHpPtot=pS5O4zuF=()lO#Q8LVERrYO{T8<%5qO!rCdcTd9Diy& zC!{vY%$8?enEN)b92Nl>-FXKbAROmvSiFp3vPTh4Zp%8Z=gHJ?K>4LnS(xX zb1J2eqR#`H`qoU?o?WSpX&zX!Zl6h?>1f9m)UeGc;iMW6RBN` zkl5>1k(4{f2WZS$^!@ zVpweBLB^Ep_DMvN#Cg|2ESpjZP&ZnkE5Kw{5h08s;?v_4zyDJ|M$~D7c0cAczQA*4 z7-2EajlyM1j2A#K9ukMp75cR6ap^JDiQXLOfGnP;iKT?t%&rN zW6;7Lzin2>bV9bWHd_5g$q_v)YP1k_D3ry?S7nHbVHw+=Vu%rQXzPTumK-5KS(A;& zI_u;DQK;-3XAyWu3o+}Y=g>xm!&@972xxn$eg8vt_ardqgyqTVVZNKWmp~shY`27o zq69i#6~P3nHkYnM3*&Hgg-N^nKa0yH`ZIH~mG|}O!Cn<_vP~wR2{kxG{GrA*RvY8K zNliJ&ZTjR9^!=>yVcRHcZv-zA@5v#vp zYI;Fe6~Qj#_X>8S-WlaX9v?%^mbl-kD|H4URprK77}T9wsfA$5#tsiB@w$k-t2hKB z;ckC-UZ+?VQq@0oR}SiGfG>2g(5`^|$T}f|n$P~Ac@h%RoBOO)QL0U-2!P%wips{d zsLED_O>Q@N_{5ZgmBp$Ed)<`1hO#8uDj?_B5&nnuv)yVNfeq``N6E(e$>H=yi3F5z zzy_?9I!OZLvNVpiytoK=mlJ$ulD1hq-oO!53_>++*WX4Vi1}7lS2-&}5bnJ|t~%?P zMTEb0rtS#zI2RWRHDwG}h3{y=&8>Qr>j)!3u*(hn3>26YL%s@o1my&q359knG`dqz zyk30)IO4i98PPUCphScEb9w~$piA#FaT3dUWv)o-pKBElLM2yeoAZMUpHI8uN>O^v zQO}LjuKd*pH2f1}na}!3u<@8A)Sw#kF4XQ89j;t)w(`+N4DWmn?d z2+i9@u~Dq`ZcPfB=IjZG?h=BI+}}+Lkz+7!EQf@RHL1(nSRA_-oF9*2U@q;UN@u$VVQ8KO9Bvr|F6nB#ZEtf8?B^8gnu^v z`WG&m zKEQJZfNxH;;~fY175Z-j&m#Hdqmg+R-GHQ&gD9o0HrWqwurD#R-ySmGv-=HI^c1gD zwYgO}Sti&zRVeh`-zlbaJxPTod6hmnHB5!=sR#GjjqI&I4d)OoY{jCiG%^9l`AFErDd9JHjyE;%@t zoSwtaPOn#wVgJ$`LE{=$fGi zEM+B+x6g!hLK!By3B{`z+qL$?*CYWGQ)=tpGL@Ll5^qV$>QEAfhw)z*nPM$S6o7Ih zPB}WUZ@V^N5TKb_H){!DoiWxV@iZfO7$#tj3H9|I10G4Lhw9o@v5pZbX^G0yBDE>E z|M1D4{8|9ZO#=&)kYbd&S{&&EXvJ17;={#0B+H$nB)%W7rnMddqR5PS43SNe;E;F9 zE+QMlb_G*0toS}$mRuj?y+v7epym)z7Z*RiJ!kDUyGx!0q#goF z1lU@}erQBm5)WJ5R6+dPE02?nDoLSL6FPT->dG~d1yVNUjUUQs69|>6qQVy4P6z8egty%z#uULZ&s2jW?Fp4iQKq^Wz~HyVK$F)P4V7DLN8Y9!+F`+s znMjga|7GOl_+X&4Yx4(;8>S(-{XLklu7{62fB4o9#sA7TODSq|iYeuGFw-i$ALJwN z?YCcejmWGw_gsJF8AQ>ZjFHOfvhE@DzBc6g;L7?@tUm03-g@0*e(pV7BHJZZakVL5 z3tSnbG5*j7S#h&Gcvoe0R%iolUJz&v@cCRHj!~e_TnU>%gmH^RUWYh7Vn#D;m~?tb zuU91Uf>ZwMkbrR1IxN3DxJybgk`Q>`iCW05&3=^63x1eOF14G3enY>+)Go5O#604MLGWJRHEUjnhpxgKg>cKb(@=Fvn|J%$(wPu(&hPAF+r8GnOx ztum~If0)stw2OxFuNI4bk5~l2m}^0?mriM`sW;69uY9h=Vb(Cf0*U2sy4{>uAw%B zmEQ2}+14&Cu-OiA%TFJYs#r*A z)qynN(DPUDsJJ9+gOoC+Aw|wc5S6{q50eQ*p^r`s1kvFnwbU|(kkUi$93rM%zaD9d zKM;gFhj z^O4=x(ZU?v!;*H1)LPX`2>=bGQEkTmJFYLS`d8S(-k}egdju$|jTk*FF*uV6U_@|@ zF67`#tjJ{MOP91@OW2{?20f)Y;;NuYLPh#Mog7UaG>(aKx2S@$}hMGhrsiA+qxmf%?3EFI|SAoj}a zXG$P0BuNp$3bkPn{;4;MdRR7kMEh=P=&L&u9D0*M-p~(25WEBby}_n^<(CAU57?P`b1-VsB6%%MEM$O4iO*V8pw;o%2mG0`0C^h>T2oiL&; z4R&bCN1-EepFzb5&1BdoF~aR!R0X6}B043_OI3o-I+&2q(r7uR4r0D>!=QmuFB_dc z#Cd;!C_GL`*pmI&4>q!5tu`kwjfe|zTvjZtTG1)=g;$!8TSusve#Lmrp?xxO6q0;2 ztc4FD_D7#paqN}5hyVE8I#)3PpWNT+=kcq#uruz8Pg&StbtUaE1Dv%t&A)%2qlZS?rpybhE#F9kknmP+7Fn%e(p)Bikzjs zcR?+z>z0)1fs*;ryQ|0mNP;L{`(hO2G*75pX;+M>D%f45d<|;iFVkU)W7BDdCoHDZMKHGZ}fDahIn9VA30Ww&x=UM=SKb$wa>@yLXOClaN6gNTHP z9DQv@Q1xLW;JQ&`4&{bmhJ4BgRg(**gUYT=D?zte@jkjMMa3q3vpvVv^M8SRBpUl2 zLpemb6)#?PfMRYM6i*kHj+U?%`$pV0?7{x{_$IY_oK@u zkRPiRQarhDiz4%x$WRVtGT*2`$(^|mp!j-<>F~%-hSAHL!e7lv79~ymqYD$oGd}~T z=R=M&7xyD#K)|>|OF%;A~C=c3?=8w2fzC4ql1WI<>&7| zi6DkQ?l1Z<9A<3{Ihn03Q1PPFrcD%B04SP1R7AY5;9mXqSsB}?!;SRcQB;W1l1+49 zCkOOzJ=c@$Gv`Yyw%v9W&-~bM6XTwaveGIflSPj82XY4+q$E4hO4k>PuzU$I5*T=* z0xBms9%iY1P9R$;^u2JzEj2V#T(%A=Tq*u^&kdP^-hMCmgHpsS%<+F6QqbD_d<^+2 zq&&gVmRy3$wWQod1t5diDF+o(>iFdl+UU@Vu{EE6gwipxWqEJS-4zp>)t+s z+CmgEpIf%5cVC5QEM7S;&F zpNMBaX`fV?!T{488eyRFVNiT%K0z2oY|(O+v~@Z_7nW}3Nx zmCsSsxh)?|OR~R|-qkyEAi>n?Lp$x2wsrgVwa~IJ2yx zg)qj0`!t6(&At=G4uAmQ0mNC-E+nFiNfqr0baw3p?4DvAIAwv&#u3OmDi(ND^by8T zEVKXBZi%IJWhNr%lMy*J+w zbV-M7DA?|fpWc_gx{Y5?yTCy_hgBu0ex(2j`f+xKbP1SEm7n*Pa1r{74DruGNsbVL zTw49`*DnkEFstbn-5>JUWa{LCm0L@QeZ1cOrfa?3T5F%eR~bs5QGO*Wckg~uFtwW| z`Cm6sdO>HzPn}}t7n9^DRpfn%JJn|rX!jD&(yLv~rmxBWEZ6n(3m>aN+TM0m-*t+9}@VeBC4QW82|tP literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-static-no-content-size.bin b/tests/sample_data/compressed/html-zstd-static-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..3d494192e2c72294810309eb14e8d55e0431fd1c GIT binary patch literal 8063 zcmV-_AAsN}wJ-eyIF~>I5=%-jHWmwButy&FfHONt+7lRiP9buC_okxG);TpGzz4dXqx1-M#8ky6bI)x zG!o@Bi=#bSV_BRAaV(Yzlf{Gykw(I_$)bVN9t(&9Iv#{dt=WU^pFYb;7T5QlP>$ONY?N;{22IS>mtaNrz=#bW|tA_kns zqAbfq!U z0Z>pT8WEhvqA(&7OFPZdnk6D}9F0=pFj~_n5XrJ&nzOVPaU=`JX_f|=NJJP%qBtNH z7^Eo}1;xSwtAG)SP%Jzs79JFmM^l!CgCl{&LV@FuSS)90k4KAGK(O{`ltyvR0%I(i zL$Safkj9}{U=SieAaEWq00Be+3pfBk5lC?}eXVd+jp*mfb+R!RpI3#OcZ=$~8lBel z6*1|ee@8v9ks_keVXD`PnEv$^Ro9plF^v-4qOLP3>K$n&O*Bfnw$eoWPwBl%GmX?< zT)IZaU7G1Ty}P|Ds&mn)dJ(@)Uf&P>t9*~i20*$Q%YQ4C$9CUiHr&rS8s1Z2=Nw=>mM!q3-u5#y01^w{Vn2J7d_#fl&-4s zVy0Wfb$V_>l=xNET+FY!#f5A2gvd#ca?g9Y{^6=zlrGb)N_)|ZU*|e^U00zla?+po zbn%`BmUF!3Brp)MJ`>)Lxrpm$UOgrLN2{wm zO=#WYU42p4$HQmpdVOX7w|RFf)0e0B+f>}kREczbr|ZAXg_vHqsdTM*d%Ar0rgJ&< zd0#K;Y*U z4-blkg(p;`jwr21^mKJpyoB_e{v$#N5vBjQj`?)n=^qs-uHz9se?(u1(i1yGnAB0C zgsd*@J4%#^M<`SAPn3v~L!v~a(owRMcSe{Ap}b7YqjXBDD4o>5P^wf;>WDI}g)&n@ zC@-aADD%>(2x+PnQ;3rKLV8bUC#EP9`pKCollqd%l=LYRQs=Z1Q-waokSfJeDyHcv z6-p{mr8`Qf*kVap*?Kz_^LkpDs@VEUnVIX5V(WKm-K{dIFTGNQsnlXusaRSp6}#_O zg_^h6x5n_xnmJB&0#25py#291I915YC zqfsc+C?qh342ssMM9{Rw;LIQZ0Rcb&0RYnz000;a42X20~#l7$_c8GzCrqXaMyi9mKXpA=~z=5%n*cSe7rX87Q$kvpl+R zi$nd26FGbBNw?JYU?y?JOb4oMZ==WJiFjN(t@6?q$KA&YgCqricy%w~G)p&ms9)~q zCxU2^e{VCN7DB~ZnRBNaIbfq0RruIRhoke06nn(;lz?>vErQv<#?(;hQM1?>m7SHQ z-mmAKHWeur#Q}UJz!9YPB=c)Hr;KHAOcoWPBEsP|1ZEU`2fIoG-7ooUn14-narmw# zLHjr}v;01&ZA1Bul&j+o$8%l3SQ5d0Nmp@zG!!3U4FSE3Fovy({2{*kTT@+km1$O*n1IqEJPuq)ZJA22@vBBl zig1*0;1?7iNf!+JnJW|{*qeN^4 z!e_djZo%VcYR|!GqdrGaSYM@6!tPi5=8+jQ0?X=S!<6n8>IG%IfoSVP)+iIyE}0OV zL#OjY&N85nN_kp=oNfWz(IPv-#upgBJ{OU4;_F=dwc&mburUsT9QqbC6?cSFjTzyv z%CB#Y|A}=Rs%mBg{+PlsvEy#xZkh2XM$evI462Lh$eyse1hw{~e0&T4wBd3kckv%C zS1zz9`v1S-@u}ZK{&Cu8R}Knq^WMH6f#C}R8wG)g%2(+@%U}(KxTyUy+*Z=FS4lk= zb?**ykLamaRPWa^VQIU>Q4?(4T3rNzZWo~)OT8Y7EjOc9*G5Z4Q@v1Bn*25uqKgP^*e(K}eiT`cgkh`#$itroy zAjCrDr9eXkUg4Oqx;okXoBE|%iMbMM{Xlr=qn1jw@3eagE5kwb^M54o3%Ll*<#YtN zQMdjR!bA{fw^~iqGWMq={2^f4#lrPLVzY^`^JKv9L2ZT+WsEZ%B7kGCc)wV}o*gOU?hhER z(e;hOOy21cMu(hj6s!n6W}fl-mmI>BnFl_7G8jxc9K4%m$i-!JNfy+zSCpnLvqN|v z^r}ug-6U6$6o=_{vfJ<8Lf`R_sc+OfBbe{eeOtqshfV7I9*=~W9=;ZRD6fYsPnqby zR}XJCr@x#N+aB`HBR|wkyzU|Me>zscnt8<1=}rFuIw*BxNvD%N0_BihiwOxw>&*HM z(|6#X19f8)w5gsWklxR1qhuSP%N*gbaM7YA>jPtCzGIFFrX(#6llP{P$EH<6!11Pl(WF?mIPz>W|1C;m-TO5E|M0&5doe|K z0hYB?Bo||zvQ@*nx%VMxv09bfAU-V0h+DrSXw@`38f+0-wfmhLvJu{9Rtu&PwHMsiQX(k{YV279 zGpilv6LfwEfi`e0J?vc}8KF?I7RAL9KfC&T*fjHh~*Yi&y_`gmbPb|{tW6rmU=6Bgs$z#H;T6iR2At_ zOt1@hhWMG?uW6nNn>@z$NBw&Ai0HY(2*mj`fh>|I-2E1}AQ5<(ye7x!r5t~1JSU_! z%FLE$U6}heuN)Qu7~OdX93UL$YFNCCVX{XNPHxLOuII_-Ij!pcDDe??PpG?5vGr}a z%6Q#2oPJ2($TE!))I$4I?lq@zMSP}6m1APoLPoUThI>>n6L|`=G3`4QESZBoaC0i9 zkEDBnbw<10xh*3#HD!-bL%TL((x-A7w=~G~UHL{yAqT%ZDMfkK;kXs1qB!tvQn_B4 zr2?47!_e{p$TPm30^E)#6F1mK9qv?GexK_G)^N&SX%D_MT*-C|g5 z<3Yxh>-I@RlEit}K`fh62~am$p)0^-RuLhLBI48I6uv8EZ)rsC5=zuJ)V)vC%bTO(_5x!GNZymYP+*p{5sW>zK zVlCJxp~bz*xO4QZmCBM}3W3}ik~XGxW=O```o=|_z#wTcC8_#{8?#HF?xr&SvEk80 zTVo0$=TBE>qP$>PFp7f|V|OvNA4*?w3ZwpKv|QG$2#le z0#T^!9A^=DM+-6Qr038^hQnJNAqZ%DseS)LcK0MO=Y-|S>S4Z{xtBm6G;FtoiJ}BL zUKPOvt2UReL<{3^b%ja0`#+1zCHgaSvX%Gs>A_wVZ?a7$p9wWMMEs$~HC7wrzDZ3v z$8GxL5%m46@nPF2Yi|T!uSBzCIb}}pczx1KX;rC?S?=o!U}p*cv+Mfs$`Px-VQP9o zR~5l7|sVj8`B30$aS{T%wTB(I#%Ek^4C-J(7ysJ0_BjIj; zcV4Gh7E;webyp7RYJe|vu+Xl6{Kz^XgPPC&pm`D!(wqCNRZ*%y@s+R+A1LD*b)AR^|Re-8-We$)kn$3`pMz+Mu`NJaKHwv zl{!fRX6SB39r!Og9DlEHt`PP`qA! z0XX8iGa1n~K%hi}`g3{&_@GPgGjS5jd1bCh>Yr;B4?-nZX`Azd3!hKB;!06^%~8*d z)2{s02Q>T>WSP(UNwD#lB-Ef9^DflxS7f`H$92v5wUj76}mTWU}44aeCJg;)(?w-VvzbVf_cHc;TP0= z-DQuwZO-H0s9IT>2WvXLvBL-?wfjGwyZQRg#v2|vD=l6I*Mec(=YjEjD^G(WXWJXsY%N?mRv+Rz@1)8wM*y$+$(~W3^WqRmT>E?Wo@H0!+z8Fv zMzK+>^=?fHn&#{Yi0%@Cj@;i(3z1_mZY+m{jWwyu+gKdC7n~oDVPG!p%&FlqN0zlX zTE0O7qCQzpgKsz&nxK!~>gsl)iO5GDgg+cb?d3k`DF3g@JH<{vfE%r(MTCDg{`wa# zn&gn)C{ho9aaLfgiir1%4sl})YeO<~m)>Ah3D!ajhY7EUD0Wg}+7cYS4>a4n-T1}TKErjhGuA99;^K<_PeTRz@H2cmY`irh{ik-&%4x)V(1 z55C>CaRsXJkdvIK#fx%bnjOguFEg5?=THCmgHWsW&Y!+|3*Y1 zKs<1y2K_jFm@Kd(XDS&!mvbSSWgfwICDeJd`HXm_mh%b?R4+8snjEyG#x6NHmzx(UUr7~8e>!`CDM6H{vI-ZGV#&Ju4)%IZ)OhKKQA7nx!$NECo_Bu+Ux zv2VLJU=X00S~qJ6VVyD7B=IyOc^D>OjtTYk9RnUos)y>@Rk4l{DQSty(;~GgxBu|T zp8Q$>%S{6dlaOMRx>_9R18BuoE#kw)ha?~U`@y`F0G(MneLdzXhm`;)|8pkT0BgaTFy+AO$r zc|>z;ZOXA6HfQ{FKzXfD?}K%i8aao}#w5#~qa?l`ucoyg0;0%_c?^+FlHibc$}S=s z!*&HzF|7DLT$l;$S7m6}l!Nl;-H8uNeo~!HMX@7Xhczr;^(6^xc2{RCG?s^_9n6yJ zrx$FhmFiv|kG(}%cA(}EP!|_JzCCB{Hsu&U?6}s1J$$f1IRGJo{JTq@1*9GVN(9(i z#(ro-S`rUi-BdyR+bfTgjVei@RTDaQg6hgOkp)sV<&7W8X%h&Qs-nUc-Q`CQQ8HD` z{^C$fc1{QDJcPI1BgPcLG0#+kBkc*5N>Qe|7{K7S#XytS7!8$MY)9Uv9NJ;QjhRT2 zTK{F_ay-3^u9La`ryj?QLH}ffZlrDV}9;ETq4^gRdKZ`Ukh9r zq%r=`23c{lJa|`SbyjEtZC(&)4eyUtO)H*D`Jh)3rF_I8?-;0;u&*kv(G3bz~Vx01q&(qS)Y5(p>+xsJ08CjMw zj96&m3T@9*Isi5zobu@pzkOFpkTK}tChPK9PY`x-`%jdq)M69fH}ByLdR$Yu9@Q=h z+u?sUMS+N?L;xr6qhv*)ZC?Vh%DEnDUUvIOljhMxRXv6n2~XWJAxd3TLBshLF-j?i?bfT)!S^ia!v9 zb$N?J+4zS_k3~knp*_PTrzkI@xN2Fgdk6yrUP{tZv@y)$V<9LDydiD|pc|yB!t;^c z*U`cp-NTZ0iPT!vO9=oCq)~0g|2wWPt@>Bk!rq||ntKE&s*M;uEHOBf31CEUjV|Qi zORUIb=1Z5fU`ya_?aZ}1#Abkf&@814#@Mn1!Tsf&W**r(9XhI9JvgO5bf1?|kKISBp67iH8wnXi zGBziPrB~%RSJ}N_K zE+k12!3wou5dNt*i+Wf#dPMtfYUryw6C8SzLEg|0LlDA~+&gK6$U@fuv1B(i6WP#! zbu0RQS@)pN+9kI=?ColY%ia+~eaxXez{moU5ZBW+S>fRaWiinl?DR{n6P+-kEe&>P z%15Ciai2lO3C(2KCo#h9TvP?5RU$ei%}Z5+&N`To(9&o*rVe7hal@d2QZE~wKE!!{ zfG9jpNZ6A7*bg?cVy!kOFO7%`aa>j`tyF4pQxv(?ticeYCV09(!Faw@v$ngXdK$~E}R~O#- z=sZ(Z}-Z^5+F4B2@(Yve207!x;Ui)Gca%cnTUB*QwczeKQ+r#6y7Ld`OIPXfsg3;yw2fcK-zCXgSi z6;eF8Z;K-HnaEHMWisEWK*^oC51{yZis|skPKMFTo5EksNfsqd{G$sK#WOzxr{_bC zGZ*(GVnD#SLrXwIW@(*}E4JNs70>+Ga1-O6jA#V(Ms1Bim-eMF%lSfq5>)> zI38xHeNG@-DfGQ?#4R;6Q(U$VDO@T3bk7Z$g5G{F_=8f!EX?tL9a7NR`+N-fDx^HY z(Ux3-%C)Kp&Y8un&v|F3^NC_#JkvU8u$0fLK4OW;1Qa_W!MMaSxLLSdZ|mMZg4#k9 zGM`(vsCeouYh`#%KB$zT&0vbp;x2;M4MJA(;U*~3np@aJ1-s-{fPUy6K}3pzLe|67 z;}D^W4%}L;ENmB3|F2R`y2p@GU`VDWvV>}7WpHM~ink?_Ga`9u8YPGIa~9SJ#Gi;~ zKWU#-nZf|m92#Mu@?lVXXg)z0T&wb-A0J=?DH(CQ?3n!D#j|%!#}r^Bs)|iw4GZ?q zC4JPB6q*!Of!zHjxUfYtC8C`N(1x2%HdhVy)@;#om9%v_Kp1s)8xK){`evHBfR)ct z)a3pOMMWetiqN0)^p(rHABV!>05>p)B#T)LJSWuVsl5pJ21UKd-$O4-Os3!@(=Z$a zdtysai~}M!jCy7L_ts(qY1wrm((ihE8CMER*1I!vlbb*G8Mmv+vV+#N?>MupqJ=QV zgZngxHqE{h#SVY~-~q&0(k>*Tj7b&k33PVt1?-+;95`ix&BhVPIw}@;RP+(XP%N|! zH(RO|3em0IZ)KNlmMD}mJPF1r+sPXP#1qSi1*lgj*6NX7rpoAmp-G3$BE2`?5Ohh0 zY$(|7j-TF_zPgQHPrJZDJcm^!sD7mY3Hot%hI9#-O_iVbmT(dJiVX43LP?Ggf?Qht z@YgR3`!K8N7Tq86*ktPDf|XlKh<&`?{-$fa-CAp(!dDqepHY4#D|hdHQZTifCi!1C zP&}Os3O%?TsHs! literal 0 HcmV?d00001 diff --git a/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin b/tests/sample_data/compressed/html-zstd-streaming-no-content-size.bin new file mode 100644 index 0000000000000000000000000000000000000000..97bdbcae01d8838c3ee046f227b91b99893daabe GIT binary patch literal 8047 zcmV-#ACTZEwJ-eySQYyK`m2ZriV$moT92z_% zjb-wPIGDz0$^#6}5>XnZX%w0S38$gK!b~Pkn^+oiz+qbRU~nFb37FO_2_$eD8ZeDz zY0YF&BAi3X(i#fGw9_1>DUQ?79*>mIZ^uctjkM#yB7#PQwy$Gz(@C zA;M{B&?ZbEoYq(tj~1Z-X%GxfV~Id`s6euKaF_$(;lXJft&tq1MJyte#?l(5J#d!E z#93O?D3YZ$iy~<(%Azb7h%=cut?_V3NSKHN6O5%T7F0M5jb*}ElnACda2Dk(ZDLt8 z62-x3OJg*}At7Pf1czx6XK9UP(nyr{m_Pxtkf3SW1E*0S%|z2Ar!^9$ou)WA$DxrZ zr&%2B(HhI*G>BudOqeVtOo%iRrcD+Noc35i6bcrQWpSFqIH#cv&`=%|k_7`iAd|=f zrm-}KM3Zn@!#D_RW%p$t8Q`OT0J3h(xcq-Uao(*Di@{8bgR-{^y1gK&Ry44sEeHR=RIBgruQ~; zeYfz7e5(-Ei@2KWTik`}^0=yt$&3Ge7vd&+m2;6UE-&7uT;r|1m&tem^s6*^jf=)r zxpec^*1h~DU#>#9ZyprqaLE{$)+|kefqgQY0n796X^sO~Hoj#v;is_`(EmFMybftyrrym;c?pB4qDk3~8RR2pCDs;Ng>RwcL zUCNkuG%jD}5BD@tF&)x8I_f3jIiHv`ofgqbab2!kE5-k~=)4sws`FMqdTAX|w3+w@D91jnQg@q$h zM@5v@BYL_zDqcc*PX7@hgox6AT*rJm@AQv~6xZ>Ho+Mv`>uF`GV(TkqX0Ahut>39rt4!)kuT)_wwb)fEmR4pSyYE+pnzz`xLgvNv zez<&Xa>A$`H!?Lb{f&#gKzu>UA+?m;MuDM@y+{QrD&Ixzd`$BOAhLYeI2144jt1o zV(Nel6GP~bXvH5xjImn|%J8Th&R8PE7-K9Ma!80V24aaZ#-ceCLNiCBP^M8xU<{c6 zW6+A!s8sN@wcyMk0099&0098g6951h3=D`&Ljo#o*Z~xP0U8tr5ELv76bJ$YKycs) z6R{{100u%~Fc2snR5S%n0%!pBBOPGdQ;==LD-nJdO=c4|$1C}PGP^V5qHDG|4zGZh zGuoa6mpUIw5*KB$2&)LTzs6=^=~3KbQ!DjV8nIu?cXB|a zEQ>?>3gr>1_ax;Pa}Z4|B4ke5@^yrrZ7847!w!y0bF*K2HhgkThvLXtO-T1~R3KwN zbg&IoG}3s-AjgxqUydGO{3V6r7}qdy1gMGYM(7M15&82CbX-9nG({<+pM%j~9B-KB zA7SrN;DeP%1k~n7H*D=gSo*N+gJzzpi!at8g7+ae9)X{_cQ_w^IFG>6FS=Gv|FD4H z2%82T2t z8JQJ_s^P=o*v64g>*6U`ja}~PhuZ|aQkJ+aFY76dfyNx#&Wm+|0$f9}2I-MkP)~oys1m zZxWugw@wlJtRZz@ARO<|WdeFQF7GFjSvE?2eLK0|rX6zg7Jd;6hdxcQ0Db9dsGvI> znO4`eHtVHcs8)$QOD!Lcg+A_5*%xT}p90Hp4FCL1l6P=64=nrA)U%VV z=35zSR1(z?&~mYeUqH09BMhF5`1d@UVW*675)Kx<>wE-4H=PRQ%8Zo55)>|r=qPO5 zZ!q$P)-SW0(EgqO9o&WE6+eK1aw4vN!!X`69@m1ag<|@DLj{_pwuSjCAjl)=ME$l= zghyPx5P4q&`~-2rDnW`;sTL9bf{HxswJU~3eZ3QVJITL<#^G^0yZ76#&}T8^ z@s09-1l9L&Z|gSmL=;87hag+x`<~f9H0dGRQ{KhzyTTjh>EGwXnjQ{&WY`a9YU0Tc z8UE>IJc8OIj7}f*N5MhC8;d+0{s_oJKJj8*I1YG=)0uu%_Z%8D*3_odj*xIaM>ncP zU%mE-(ZVGjEmh6oJF69W3bD2Z&uw!$SjI`v-uAI8q3xbxBmyV{f`q%s5QQ#nm;HbzfAg$!m>{kE}TSu(#Njl|&IAz+i}5 z#3Q)Z6gV2@BLKB~)D2S+UO?8((^$<5Ue!|Al-xG95Ch|{9XSxPeKDUOIbH%8TlR+XBT!c&`VInXOG;y6 z^7LA95V5o0ST4Wstbvpx-5>4u=36@y(Vs1k<15DjydRgRmTTC3e-pRleQMPx;@>dEj8t^N06a? zn6c1PIgT497foJ?Hc6#`-%lxpoV7b{QB$`b@W)}yI0{Tr3t%Z8hL%u3o^kC2OZz_g zxXC=~aHnGW{dB2z7A|2bL15Dzzs#Jc0vxyTWpeFW!Xwb$DB95?X7~4q1?|M)H01u( zAGh>|Yz2zBey`i8T$$UgBvCW2=W{zx^44%_0pNzVqg&IW2oF-xwhrNL-gB^wi6k?uVqM*+)n-^FpCWm#n+ixUor|2$&oZXb z%&?Aiv5kxGf!Vr%O;T(Qw**L@>!t$yvES(0t1%govsYKwKzWhaaGwUv2c22ayt#WB^7zJ+Wu*(e`!v*W)9WQKId zO@&B)?Y ziMqTP~yYlOs^Uvj}Me0h;2%5hSPqtgOM3LCjsLEdgs5W=3#Jh12jbfxd^PeROauGXo z_Li6M>0wfpaa=@9i!3!lQ2v448cPG?9#T`Bf3yDN(S1MY@zHFQuN$F!R~BP#Ws#ir zcv|TdpsLhYQSOBbV2qXJ*@gR{+Y#K~U@<*%Rh8;4AbUl(QKLTOgE>Bq(3V8ssaSPJ z@T$stEmhXt-cy9ZjN%UGCsDh2Vpq8YBi3#Ys$b`^tZS-&S1U(=PpCGNR%ik>3}>1$ zL$?0@a6bvw(<9s4IgpuKK%U&Fj4mK8we z*v9Y&@Yy1?&9@DU>_;PF{rPabjS3wP8o~xS;+jODf*^LAoNY^l`^pK=Y>#cak5};# zc7jmOZ5IGSNdA#;rTQvGBLsWze{+Re&)|px%1KJV$&eTcU$Uq6I>QCzt9D=Sw&$)V# z$(5FWUg+s)@l2JR(A$6=95B9OoZvCNM%Oc}3|AKE!;+B+GM=ycgrRqwKhz*N=Ed9Y zR+QV!IJ&7BVBYem@Q3BU!m4fF8JCoYL%bpu?rOj&t%wOk%lbA*wD%~7Xq3oui12;$ z%H50GU~xrw+_$OH5G#=O1r8$_{uW}Teq>y^d&R-P6`&bBv>{aQA3oOt&2 z;z^m+jzE6%>z-jbXAU8fu=dw9Fi)?9fe}J&8#ANi*1I_=*q-z8Uv!sN9o6}6K_Wi` z;;~#UEytv;{YJsqH8|fW!@zIa_gjPi91&@uxBL$PLPR9VG04GO?u?m4y{=WMCc+5h z0WZVRZLcJQ4*&eBsZ*r$1KbFmXhb+?+*+VH4-|mObq4|op;axBTMqWcr)*@Q1?w_A&GblrN-tsp{*>izh`P0zL#Smnq?lr zc_rj|v*C?6rE0hQbneK3z8N<*@#oV8#Xned#K@uxTEbkwWsULZ%1T>7qwG1 zQc0DOA6j>sVs+PwHIxS#%+U;%1xpxSIPrenvvWOwWB>Nbp0xb@A zi!i~G#eTT%km92cJ{VLbpct0aOyG(Q3fKQqaSfQq$M#6TjO3#Z_R`kpg#=pK+O8yu z*u;8`_Arg{NgO_X-VbI(ls8tTUs~F*qDX|Jb|AzLtE4<|Hs$FEmw*f+oBKopr_9>cMdI6Taqa+Z+|v0Z^QhA+Plnq~q&ptiL( z<*mu_?uSE`pR{^Y#o5utA)V!OwEgxrv8|+XRZGDypy@cDd8TxJ(7Vzkn5!8aLfq4f@WHU`iFe%y#4eGy%U)==bi|zJVO-mkumT| zT=qX~|7)aN9~{LW#czjr=&f=+W~T<1Q6l>#Rk5`xXH`xa%$e&j16i?H9+<0M$RlXO z&_!=87O3Nq9$oS>qhW7F+(-Q2m`%lJHdS7MsG}^=TcwAFmgV$ZM5U05SJ8V61i2C{$@>xd0sE4jHllVZs z;34hodJDxMFH!Y5wpluqaXb5~Fvu~>`%RF}_714cS(RiNe#b_k_#$tTg~H;u8|_2C ztpibDP&eH@(IE)yYD|lRY1wU;&TMvSxaBAhy9q4Bw(2vze0cn^K4>nv_8`R=Y1E#x zA`qoe`9t>uA@ZX;0znfvNr?OzLlb-OZ4S|r>*Gc`><=(&UC#d@y#68XvE+z8+}LoY zDN@KNaZ*d8Jxr&8mr7d7Z;ZP7SeD5G{)!tK6oYhCc!sh2Kv|gEa`;>^a^U<1hhAU?T#DB7>uC0x%1%zCwok5|#0rd`V49lM+~F&3aLCxIzm8t@VN_rx|A z)M9PkAMe_lHgafkZ=O*-Gh}O(RlzecPnB&}+%?8}DB@%K^o+v7G=va%5?6>OAxbd8 z;&e78YyTw!WOLrwp$yCn36X1*`iL-tFk7HZ>;q(buf0sxAIf>D1e+)Y=C*~xu!bBe z3`IZ|1czaTvo-a*EiG=)kH|dV4kAu_EFP_7p2}+wE@YU>*tH~JykQ@P0F1}6cfg3P3*7-C$!_RmvZDd&R92Sv`hDcK15#j#=QFzmQVN(&MADm#th&Oj# z8sR9!VJFzwyJFM#MXogXx9*~thKe>cC)H#sDI^GJSTY|M#g9&?;xjJk9{BIQb+2MN zKDjm1|M5ZP(AH8FowTss>q^_o3~+g0>X9n1cCDjIXxe{yMS z(Bno0WHAuVMI>PJsCb*-y6(5zhH2-MGv))>KI(KXA2E^r#o2+-kB*LF&pBJp%8)Ma zgVqEKES{3(s(Y$8;U=%B!16tl{b?AcO`dybs$%(4PhC)1qPR=?q8YyV(Sa)l{BjWO z>ApBBDa{ivSJ@R4s|tG-310)6hn8s}0oruht1IH5JAB((6a z)JP#89c#tL4yUl(-gmCv#PR{OA+uQsZUoo16|GjH^Wh;3_E(^k+>PCzIFeMEN=9#7bCZR)inDx1xjv-;l|1r};lj9tjTp238If+)5cQH^DPvOxzmP%HI;R zO++NsOj;2q`Hh0B&3>^be@j0)AVqC;1R5=fAfhI@gbFhbl4mr66ye$hoD|TIaIUZl z@a`rf9?03+uLo-6Ig4bFsaezClMv-62LFy00Q~65Nan|`FcjbJ+oDL^Oly`S8O3f? zu*`Pu0w{VdiZOX4t;5LLO{Z4pC3BS~?xUNDBAFk+sevK4lZ!h!F#u@XaZEravw({x zfr`nl1}~J!*<_p$EwLD2D25XA<$+%P|ItH0u>$kgpG1)2AFo$^_!?$R4E@Yj5~>(e zs?#P4Jb)LPu9PCOYr#4F?pYZx)bU1o-cdw}(#a-@yVFGUXFbTOYR-rVBA$a&7B z2ufK$VZ8K4e8C0*V|)f^mT`c=@>OM{AxRA#EWFnXW5YRWwJIu`s+$Xi%Nd zMqrA|<6eqa3_@1;FjEvIlUvxM0#qOLAA{xpb1@!oQX(g6rnR`X->_& zAIFWu;cj3INrtr8e@-YTsYems4T^D*e~VtM8&JVX7Q>(k@Wdsds73c~R(fU6Zy$^O zwq>_SWccvFGR_J1y?1BmCO3ZU8E&_nWmkNo9p#L!ilV|u)_122ttVV36k-6_hlgQ~ z#$A3fGCGZPH{RJV1uWf=@sr6;m>BU-)@Cf&R54^2X*p@*(rjNVC@ifV-IDFnY&DSM zPW6mg-^uF^l&6m22C%n~o7JPi<=NCR!1Khpss=hTD#`$)reM9NkF+URcs3JH;VtoJr literal 0 HcmV?d00001 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..63a69f7af 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -20,6 +20,12 @@ FORMAT = { 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), 'br': ('html-br.bin', 'br'), + # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin + 'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'), + # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin + 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin + 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), } @@ -80,6 +86,26 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b" Date: Mon, 5 Oct 2020 23:55:48 +0100 Subject: [PATCH 5/7] Minor adjustment to the test case in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 63a69f7af..7e9f9cc5d 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -101,7 +101,8 @@ class HttpCompressionTest(TestCase): newresponse = self.mw.process_response(request, response, self.spider) if raw_content is None: raw_content = newresponse.body - assert raw_content == newresponse.body + else: + assert raw_content == newresponse.body assert newresponse is not response assert newresponse.body.startswith(b" Date: Tue, 6 Oct 2020 19:51:05 +0100 Subject: [PATCH 6/7] Minor update on the import order in scrapy/downloadermiddlewares/httpcompression.py --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56421a6ba..f504302e2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,5 @@ -import zlib import io +import zlib from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse From 156bb0a1d413c7c6acafc30003ef98030a06e47f Mon Sep 17 00:00:00 2001 From: "P. Chen" Date: Tue, 6 Oct 2020 19:53:40 +0100 Subject: [PATCH 7/7] Fixing the minor typo on test file path in tests/test_downloadermiddleware_httpcompression.py --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 7e9f9cc5d..4c5bfc577 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -25,7 +25,7 @@ FORMAT = { # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin 'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), # $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin - 'zstd-streaming-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'), + 'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'), }