From ee6f5c56b287f14b175fe88118402ccba98b0571 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 10 Jan 2023 17:55:35 +0100
Subject: [PATCH] Support multi-encoding Content-Encoding headers

---
 .../downloadermiddlewares/httpcompression.py  |  55 ++++++++++--------
 tests/sample_data/compressed/html-br-gzip.bin | Bin 0 -> 4050 bytes
 ...st_downloadermiddleware_httpcompression.py |  40 ++++++++++---
 3 files changed, 64 insertions(+), 31 deletions(-)
 create mode 100644 tests/sample_data/compressed/html-br-gzip.bin

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 89132c3bc..c0489dd8e 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -52,30 +52,39 @@ class HttpCompressionMiddleware:
 
     def process_response(self, request, response, spider):
 
-        if request.method == 'HEAD':
+        if (
+            request.method == 'HEAD'
+            or not isinstance(response, Response)
+            or 'Content-Encoding' not in response.headers
+        ):
             return response
-        if isinstance(response, Response):
-            content_encoding = response.headers.getlist('Content-Encoding')
-            if content_encoding:
-                encoding = content_encoding.pop()
-                decoded_body = self._decode(response.body, encoding.lower())
-                if self.stats:
-                    self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
-                    self.stats.inc_value('httpcompression/response_count', spider=spider)
-                respcls = get_response_class(
-                    http_headers=response.headers,
-                    url=response.url,
-                    body=decoded_body,
-                )
-                kwargs = dict(cls=respcls, body=decoded_body)
-                if issubclass(respcls, TextResponse):
-                    # Force recalculating the encoding based on the new,
-                    # decoded (uncompressed) body.
-                    kwargs['encoding'] = None
-                response = response.replace(**kwargs)
-                if not content_encoding:
-                    del response.headers['Content-Encoding']
-
+        header_list = response.headers.getlist('Content-Encoding')
+        encodings = [
+            item.strip() for item in b",".join(header_list).split(b",")
+        ]
+        if not encodings:
+            return response
+        while encodings:
+            encoding = encodings.pop()
+            decoded_body = self._decode(response.body, encoding.lower())
+            if encodings:
+                response.headers['Content-Encoding'] = b",".join(encodings)
+            else:
+                del response.headers['Content-Encoding']
+            respcls = get_response_class(
+                http_headers=response.headers,
+                url=response.url,
+                body=decoded_body,
+            )
+            kwargs = dict(cls=respcls, body=decoded_body)
+            if issubclass(respcls, TextResponse):
+                # Force recalculating the encoding based on the new,
+                # decoded (uncompressed) body.
+                kwargs['encoding'] = None
+            response = response.replace(**kwargs)
+        if self.stats:
+            self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider)
+            self.stats.inc_value('httpcompression/response_count', spider=spider)
         return response
 
     def _decode(self, body, encoding):
diff --git a/tests/sample_data/compressed/html-br-gzip.bin b/tests/sample_data/compressed/html-br-gzip.bin
new file mode 100644
index 0000000000000000000000000000000000000000..57d935008132948703dfbd2157bfcd2f64a88fb7
GIT binary patch
literal 4050
zcmV;@4=wN?iwFP!000000|C1aMDSl!Bry_Ui&!nakqA4OMkN4hnY^*0Sxu-6Onw+O
zKen|n&_@R^x11`?UHB&Qdv!Xke+to9ngT^wtJq%<IOm;C1;Y;5yb0mb3L;5T3Z}+w
zdR_z5EE~n<VtX`<;r;!Xt6Q?RxG1S8c3m~}FYmvJTSHW?lguNPx`gBe{Qo=uH7Xd&
zv$ok5Q`oueg|)adf6Dtq4)bFJV^#vV$`lqj(6BsTEh_;4uPz(p#w_>|MOrZg%Zv==
z#^o6gh#VnQu%QqFyu1{XOa8po$-rXyfeRcL%J;v!v0y3%;;MM5Z1IIOJh@vey$Hrc
z`cyq#UGSLm0f#fetq&e={Jv~NOX14rvhUEQQ<rg<*Yh8ji5WRbjoyK&iiR)hU~P-3
z)lf*f{+G$9)<=7%UCC|i7`4CqEO&LkD6;rJ68;80mrw6V*}x>kPa_{t#^bL8eW#S;
z;XJAludE^^Xs7vb^G<FLDV%cz)QNENCVonLw#59yn=908jrXp2aq-Awd3i>$=2<6|
zN|&rWH`{#%GOe{%k{{DoUo_jh<H;Xbm8}R0^7xi|G>zno5^D;vEJ@|0ostBf)Hx5W
zNISxB$2Fo->b7*WeazBW|7TMt=b=nnD8*gAu(T_GwV`0LEBZbyV{@bHf-h<@+TWnn
z6CrIoPvf>sSP>1Y={-EVepzq(bT5^?cS7ZXrYH<y2L~&gVw1#uvp*`H{Sh9@Uhv@5
zS!|?$-h2H$D)VBpNzaL+He2GYioYHD@dz<daa%S}duL2jc-YuP-bKNMA6nWiMU@gB
zXn%y8N8F+5?!D(eOqhJMSVF}{DQ(a7=ooU0<g7dOUK_7G(N-S)y{{hrna1W&B;-Sn
zaZDx>@_O^xp8oh@m^ksvPx|B4;3-^8Eq<U!=OCETcl*NNmxJ8EBdoc<xwvlJd$5?J
zH`sPC+hD7+cFcH=tmkSaAuB!bV%ri2=kE+Izqn<s*Y1^qRUZAQ+?L+t8kUXeRb@?X
z@pir!0EW&k21c92DYeuqor&p_E^u>>ywvE%XvJoA?A&MId1)NmDqh9Knhu;w6CG(Y
zUJ^Guw$*Q?&t;U+1Tr2!jbm_Q;bOTP7DM2y<NdeXWHJ7*Pwy%&-;`3nRD+FSg+lLi
zB}E+X7&hb3Hj5?Lxc>)Jig^7jcdMs+GjUz2rDC{4sXLm;CSJM)BVO6Y#S*k$V8!;K
ziN_Q#9oza+;Kr~+t+iao<ELR8TxpAbpTZV9I4zyi745hgTP~*cYO`bGzfR}|x0ccx
zZ)=Scm*irW3Y@m)bV@@_H?jNB^?$Damo$bqxUeJ$ZH~o_$%><Ov2q3NOgR4;Z<lUN
zM=R`j1^*&lY7HmYxcPLT;~0w6UcXpk3Y=2!Xv6kS%kYrH6SwxIRv4ofox{!5jb`Fl
z$CA64O@ZUV&?zZnsN!|T#4OfF9aQ5_RJztsGaV>S4DXB45jX~{P_9f~H}~<<5#>9(
zRB>XoVu^P2T6W@iV$*Td7XvzQY)oBi$&Fi+E^{qnG0afvM7xrVug7w>!Q8d!w})fk
zmbsYT9X!p(mc7*AVoZtkzZ|&qpsASb*0}#|iwRDxS&gI{k18%h6XqxHYB1cvf{N+Z
zY0bp&Y{B0wws3~n|EIT-DzHIkx2R9nicv=|d?4bovr*1z1MWfKW70P=I37%0%1+#t
z;rK&u@A~1#Psy-`C$#lWYi)N>I*G$-i`9Cg!*OBjL`JbZ@fzCVjmNfxn$eB!apKZc
zhqgxXsG6~g;K1qXp2RU6Iu#EmwxKn}v*e)R1RG9V9@)gS#CEjCiKQu8IUI`%liNY-
zcwD{(ORO_|!G)n`oz<wItvFlMQ_1(=D*Rg7)ji1+l$B;<%Ed5LQ1%)|f;UrM=UT+l
zPD|@yhgAf_8UvSh=?dVdsTkHRbe@GnFH%#4PPDbZbd!>Iv*(G|t7a9G(6NrW`=g+I
zb=U@Pi9M?jmgZdc44vZ^gwg1UnzO*r^nUQh@QK$twp0oI77@vY@Wk=lA!iFU6*o~-
zT5Ak$EWLB!9=@eR8+@^NC4e94@AH7^w-9lR+Qil!q<&2Nk713$fvH<sX%M$hT#k~J
zUBXjC-2cm~rDRTs(iYLM$nG@;H>OO-YtcBCZVYirYp*h}xQ8*>idU1;b90ZWk$4su
z)OdXlr*#N*E{?IJd>2`Ic_g1Bcj5TPV-uG#Uf&`Z));&^ap|Fh=zwkXEIalpfm;(@
zNjtVX2J`Ass0nTsm>Ow)n6SmoIah4klJQyOxZbPp|1z1xcHMy)4jr<ZF0%C+121n3
z*^S#6#}qeJ(W>_{xWmTK=&;lw(OfKJjmTM0$vGY$N-XyYtHvpzXqB?N;F3$+&JLx9
z<77?2ld#_LE?m0NK|H*)On)DM{d$WqwXUvoTdJ70C)k&!TejgWsI{lv!<-I|c4BZ!
z)^-)y-MC@Ja%(zm@0&flHrr?xn79t7$+4hJTd2e4O|VL%&{@qMHne;RiKBB)>zc&j
z1Q#wHcc8XVgQiVp5wjV|#db>=!)go0OE<oZ&0@FVkxNXwEz~+5$4iw}Cb0GsrtWB1
zP`*0Tu+7ebRh<9R9cQ1>XSA2v_~31lRn~6c#?WC0)q&DnO{^xN2`*{J<)`Tw+HvcY
zto0-LKIrI0Cvsa5n+~j|+iY{Q*lpaHx?|m&IYTUEiM>}jShzH+?K^Nfq%~;0&CP;R
zH=2@Ryk0FhCy=KH^IjGxWF5E9VM>R#R^#P05=|l2|LrdjF&sKmupVv+i51H4=xU>9
z*((@7UA+YSTei7b(9v4=4jUcD{n!E}(5%A1jj7u@tDS<ew&0|knFW>3aW!)XjukiO
zlur|1MJRO!`dXfg?JBTu2h@j~1qRk5`5e9uU0uadmn>_Mrgs=Ty6#}<parL7RFCxc
zL8ICI&{{KbTjI4UvCS$DI-2ItoX`#(v`*Kw-z>0Ujm!6=L#<K4*mj9l8LVHw{|9y)
zA1w-G)K-m{S?ne}ddJrRYZ2RBLenY>7nYu7cU*n~{pg~m{APi{fw5m5Ivtu0beDLM
zusyx#j!rsAogHs0fz~7WKJaqm(wt_WqTI&M!r85I@L+0Jf)nUXcM9ayhnWR7hLo>f
zFCB(0qTPzn?<GK6cMm7@sGS1%YE-kPS?m^W+`8%z@fJ|83tXjOVrow6g7UjVLT8?6
zI15}D8XwMZn|t`S(q3Jzho`ZPGhbig*YXCJCc5e{bZB)#Z)iY=^o_?)iw>41CCz?Q
znhy307cNaSieXbD^G^js@8lH1eL~DS*j?DS)O4cf!!<p1a5btS?`PsOxU{Pa%iYXb
zm6Y>MIdJQ)&dKD2_0mC*9b){AJ>WODyQ^=|6l#9}#BaJcqTjITrWPRBchY*zv#VJ2
z1P@p2rvt(Ip6b0k$>FsYC8=!D*{nl0w-J7ask<)X>TD0of)5z**r^_qE$ih<W*KfU
zOt$;3RCn-gsIu*?=Q~`HllkWlET7%ljrH9`_sb`bCdQp7#6Lb?dGryR>bGQ<d!+v{
zD^sah=#9&8UoVlJTH0jW>5Xx{iQHF-Nc343r^i?9AZ|-+vUlCC>oKxeIZ@~t6FT?*
zT(2UtYi*N#)<ypRxX3k?3XSvmU#iXZ%k>f&S4*4hiQ~TFgsT$K=(lc<aXH6{ukTy`
zXIlol?_5ujvy~eMpQ~9`%wk@VVbqT8Hp$+*UPWG2Ixk)w9*o!GsrldR|H!koqCc0D
z&c}3)wNc3&m@aOgc#mXu*OvSx$2$+3^PIcfsYE`wE!%V-JLGK_+c2n}vyN2O6z*vQ
zE!(@M$2=BB?bw#(vRKdA&2*)6!7vQNIMTVUw&6eb|M2oY_|@-@rMps@u-uNDc}_3m
zYH1rTkIQ@Px+^ORAKq~6IXnmUN#EV~Zf(nUd+cMWu1#(jre(38GnaCEH}r?Q=3^Sx
zv8%3~Y`8tFzA=w<+S}{1U6x_Hj-_3RIOq+l-&w}75cYQ2vpI91PVd-|`}BX(%uX>c
zEDyIcIqj*kxR-4kmPeD<?iwX$XE_FawyM;5)~tehFV>#TnN7?-;x<d@yJzn5!St~E
z9O_*Esj^*LbkA9qHn%)g5;sQY2D~m)&Kh@3C1vl9P4}Kl$_#m{M!j>A3#Qj_xs0kY
zYRfjJmT#vMOZ`&QiM!JZ#CiX(m~+~U@m5jV+hf~?3qH@CT?JvME(^2FS>txkN=)iD
zCfC%`cFp5O=OJfYRU_@>#74h+GwmVo88w}7d+ThE`RepQ_DvN%mCS~Fh8Oiip7B&c
z+KaKt4-?L827Rl<s6-wZ9+#9Yt7K-a1^%;pLif$wsXdk4YgV;-XErZ9vpDl{8C7)E
zmhE=AlqmOJO{HY#ItSpH{x8Ft%;VmQllRWq#=Ly9xjj{DR(3q}p2w1M#jbJM+w0g~
zb<UcWRn?wCoeyA}E+J#hDyl>87v=YCOBL6B&orvwGS`v|tPk(_uz$KYn%sT#e`%66
zt7{ux%j0sY*5g{6Nc4&3Wo&9W<Ea`cb5?D8+@|HStJ-j_brx(X_lSANU8CgWnpN4B
z%VoH%s<K^c%ng?iHJz7dTvd=VX0>dhZw$vp+-h;T))0-JXfv&eo}R1O&^eu(r{-+0
z$FNPOYCPsz^1xQ-vO4#A(dYet>YQC&+ie+^<*~f3O67&A%CIUcwo6XByjdOFWf+Fr
zGAyrq%^~<_^ndFSD>t3YBm15y=egCk9k0u=U0%bq*OU*hXDMIvx(rUg7_L>?wr$Fx
z_MHD`YZH^bOLe}$>uK7rxlPAyd(2hXmT5^D@miL8XD1h4PtknMbbDN;$$lzl8;09u
z*oI*kYl+0C?>rd5`P{Huuj{2s)~v4mde(~PV_A;ZQ>iR?JT4{YOs}oZyw$O7Irn;P
zm+dy~HRp!gZFyXV+jO~B+csxyoBCdpx3fRIODt8w?W|<OW!i4HV;Wxfs%*pMvZ}!9
zjcu$k4*K4V#QPJHc@wYUl`?j<Y^LWdm&NIGwpppX@HmEH*pAyUR$<%oZmE0bOXf-3
zUg*zx`TWg`^W$>h_Lz=kd0d`a*_NCUuVLDTZLB39Ot(Ex`nm?Y&q+D6y0*)5*_O-l
znuR(KV9eZ7cjjgorpGW=hweRBO-F6tbIB>myKBw`$8cGeV_4o=+1Nkl|MAXLgq|~P
zW34mcaonoR^*Szfo?Vq~dtHXx<2B66&VqX$-ubv}PETB}Roa$a)4M9RHtp?na>I1H
z9J{JLj=fs8YZhl-rscLh<{GkLm~KtRnU-6Io$SM}Y-`DEe3)nK9rz6*0Tj2WKf4bA
E05jrCX#fBK

literal 0
HcmV?d00001

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index e0ac80fa2..ee1f48c81 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -22,6 +22,7 @@ FORMAT = {
     'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
     'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
     'br': ('html-br.bin', 'br'),
+    'br,gzip': ('html-br-gzip.bin', 'br,gzip'),
     # $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
     'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),
     # $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
@@ -133,6 +134,37 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual('httpcompression/response_count', 1)
         self.assertStatsEqual('httpcompression/response_bytes', 74837)
 
+    def test_process_response_br_gzip(self):
+        try:
+            import brotli  # noqa: F401
+        except ImportError:
+            raise SkipTest("no brotli")
+        response = self._getresponse('br,gzip')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br,gzip')
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+
+    def test_process_response_br_gzip_header_list(self):
+        try:
+            import brotli  # noqa: F401
+        except ImportError:
+            raise SkipTest("no brotli")
+        response = self._getresponse('br,gzip')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br,gzip')
+        response.headers.setlist('Content-Encoding', [b"br", b"gzip"])
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+        self.assertStatsEqual('httpcompression/response_count', 1)
+        self.assertStatsEqual('httpcompression/response_bytes', 74837)
+
     def test_process_response_zstd(self):
         try:
             import zstandard  # noqa: F401
@@ -189,14 +221,6 @@ class HttpCompressionTest(TestCase):
         self.assertStatsEqual('httpcompression/response_count', None)
         self.assertStatsEqual('httpcompression/response_bytes', None)
 
-    def test_multipleencodings(self):
-        response = self._getresponse('gzip')
-        response.headers['Content-Encoding'] = ['uuencode', 'gzip']
-        request = response.request
-        newresponse = self.mw.process_response(request, response, self.spider)
-        assert newresponse is not response
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
-
     def test_process_response_encoding_inside_body(self):
         headers = {
             'Content-Type': 'text/html',