From 16c4b4e184c1b213eafacecca0097ff44effa696 Mon Sep 17 00:00:00 2001
From: Pawel Miech <pawelmhm@gmail.com>
Date: Mon, 6 Feb 2017 11:41:08 +0100
Subject: [PATCH] [httpcompression] add support for br - brotli content
 encoding

---
 requirements.txt                                 |   1 +
 scrapy/downloadermiddlewares/httpcompression.py  |   6 +++++-
 tests/sample_data/compressed/html-br.bin         | Bin 0 -> 4027 bytes
 .../test_downloadermiddleware_httpcompression.py |  13 ++++++++++++-
 4 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 tests/sample_data/compressed/html-br.bin

diff --git a/requirements.txt b/requirements.txt
index f92603d3d..362d05013 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,4 @@ six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
 parsel>=1.1
+brotlipy==0.6
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index bcf20f10c..9202fd8da 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,5 +1,7 @@
 import zlib
 
+import brotli
+
 from scrapy.utils.gz import gunzip, is_gzipped
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
@@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object):
         return cls()
 
     def process_request(self, request, spider):
-        request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+        request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')
 
     def process_response(self, request, response, spider):
 
@@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object):
                 # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
                 # http://www.gzip.org/zlib/zlib_faq.html#faq38
                 body = zlib.decompress(body, -15)
+        if encoding == b"br":
+            body = brotli.decompress(body)
         return body
 
diff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.bin
new file mode 100644
index 0000000000000000000000000000000000000000..c7eea4bb826a1a1851d047e64dd98423138cf996
GIT binary patch
literal 4027
zcmV;s4@B@=R3tGHVT)KTxseDvm_{W4YMH#TqFGI-3`~9)H9xksFwjQ_FSnd3&0Y8=
z@_ThUt$zy9SegPwSgY7y5IE<ZP6fjb*t`kh(h4F;QVOQVZF*h<(<~ds=3;v^jN$$L
zn5$c|wzw#%7<yeb^e^wfiCaR{u9M6omAZuF1pNOy|1~NY%ColF7E{={?1i<sGk?na
zLk{y}17lVKxXKh3IMA>>Uo9&E0Ix0^<i;%c5k*=t1j~#J<;LY14~QHgRIs5CLAGkJ
zH)WjrRtE+a>mRs=<3xGiyIT}Yr9fO2kCiRXNW-0{i+Lu3MI!yIoUSf-Z226AD}r0^
zdA#v**@y{+8$OqPhc=zMEOz;5{^K$+BPXfRJ1|wzaHbBHwx3!Jh1BbRnT%?EOzxB`
zc@o=4lizjLI=g3zEFMR~-@xbc=^ZH>n1uLg<R8j-{B@x3lyW?rM>XP=Rip&%H2-eg
z$?YM9bB=&I5l()^leA~+n16V4g_^DL-W4w{9=Ts$j!|s<ER#W{OIFUs$vy*_*4jjp
zAJbP~G~2u5$sahBwFn5>*p|9)8b6}Tq?$r33#pv6Q<C75y2?Wb(vI-karLN_dQ#fk
zS<F1PzRzZq9EXzDP{dWvSlXG-Z77heqkpp!o5s2(_@efsvl~qGcu3o+r*T^*tcZry
z^d6o~&(_;E-IdDLJ3!@urZ3FHcFwJA8Ji&P$NQt=*&pGd^nwSc&SE1y^xkXlQJL2!
z8}ytwYO^KYs`xuaKOP|_Dqczk>dYC{6y7(EBcDcrg)1%XlD<j_541nR%?IwF>FWLE
zIxI2y!eR*(8>O@}u3Hx&$4JYjr`~J*l_%PUM_=!22LDWBb0`w>VTQ3!mL%lo=0kh>
z;=f_ZiDy6Qi&t|`{$gtJAKjV<!HmA!76!i@v<4n#&Hc^&b>rTHwK@8XlMYrk*v45K
zR(OuASJjF_HuS)^Z3_;}-x*kbam!jCxmOBSZRDe}&Ao{=EGz4)%9_~X?Reh+%sac7
zGuk9hsij`&OiZ72ftz#WrA9ZS6`R$ubDx3drEzSlcoi3GIyjXkI?`smByM(WtKW-H
zWt7nbG9EvTV{l{PV!0a@LvYsd{#$Oc7=PHO*Tv<VQtFp#uo+e;^iEe&#PN<{GY)OD
zSi;8rKbTU)>u0%JJ>8p$>ryS1;SQzlXd;_<=@yK5Wg8buXuZIS?L!lfDPB6Z^`*eg
zutTl2T*u?5VH;d&i+*2*Ep~8PI;ShzaWl4DOzYKVhmHR_p&Q&<N^88WHBMZTi&-i-
zZO!SFhMI0-_o3_mT>md=hBvsdBnWMe#f`~|qjj-zg?1*K|BSawH>RT%cD#arQ7^TI
z6KvdkI?!<pMQX2KEHMSA)H~X+z0)#0<nY9;eW?{@^rCaPxw_Fz9P3ze7qcli9t@q5
zGKMN%XH3juy|066{E14}8fvBk#fjm4F*<@{zzXHc<aKi&FC9_7vr82xqZLcEqt~(%
z#}k{5qy885VhzW})U}q}xHaiA*CLi-hEgZmm1KN9ma`4!uJ!HV7`SCFrgsNVv$16_
zHMoo^vHq6>mmV|~lieEkzilzWsWq#SbmLLQWoW|u<XsJhJ6KRL-8!wA7@jToo5dE+
z5c~i1R#F8v=<F8t$yzb$=!FkNTy{3fIc>l_2z*TXMh3@&sY}_3+cF$~=<QuU{P-yu
z*6@V3-f6Av4oW9+SZ%RdZ*({=Y@Ns`mM30ATfFhumQXXg(LGLFn(ENjC>~WaRuLRH
zUEPy7hC`>~;lwtyrg)YdG@M|=iOVCKn3mX%);O^=MJtD6aba>hXdREsw_u5NhA+4<
z^sKWQ6|@y+i+U>g{@V(_mUeYdG6iL&*_d)M3>B2UhLP}Q%IjQ<SlVf6J?yZGU|3^t
zX_u}5ewvD5-9qPCIP@a5jL?a;_Lpu_@^1D#@p{#)ViG#mF?W9yl&=ok;4QId6~fY-
z%buZg+=4I~JyCNO7@FP>-WWddTE~_uq2D4R*$|#Mo;&1hp{C*{ib`vZ;l|QC2kzlp
zcW8qz7Ow>GBmI5eG5r=Ij!~P~x`Wh@iT^RIF&vn>rIiM8`^4oaS=lALY>4}Rd9{?x
z2~pZ28W!2T#&BcGbi5XgW9h~am$ddO1B-hYldX6)DLps$m>P*^fkBPe_i$Q=Q0L+p
zOUieVrI$zYIdT_{Z#*_}8RPXWf?<u}!--1|9YhCgqi5N%R|(vj=t|nL-7%O~k3vmw
zv%u6y>%)XCZqB)4+m?*aBFFV!egD_UB)01g%>2+HtLY+JuQ7OeW5{mY#yF<9sft#;
zm%%GGhDL{_4vFSs8EZt&f=bTu_)ucGPgpfh2}P@v-36Ci;&yf@H5?~v3Z8`Zj(6eG
zjSk}BrDb}So@WuVuC8=js+hJX*q5eTw&5(OwWr;~oDPn5VsJ{<b`{y(xM9U|YdUT3
zn?1WW+h`V;xDKbuv7k&_sKe$>uu7xQS<N0cw0sGPqjOH{n#AD*7cL!lptewhrcGuM
zvl+?7c1sw;Y753oH@=L`Vz=RuOH8{h)H)u=OO;h7So;Z6cQh<0U!7^#W@o`F&j0C-
zv(M-=+DmPG@HWXRYd3IX=&*z8KxwWfR+G>Km$c*Z({v2&xOGa_`jLDebo8PVxh;rI
z2UgQ<wz*mCHf~JavF^>BA(pbl-m4reT$<JP9XK7*8noW#W<jYNP027`uNIsW$kT&)
zFAEg1j@#!jr9)e*@$wpprV#7@_7{j44xK4j54VKG3gvfnwb8Te6^x&*UIP9t+uSVZ
zXsvsPjSk~}Y=IJJR$*{s>bA~mr(motI4NglL8WtC&D?=w#mzb8)5KR1N}Ykemgi!-
z3hdhf_2FiLf%Ql}hp$6dS8>!O%UY!A9R`oCJ6Jks!6_NlBmI5QXm&re)=b=%c&$oo
zvx-AU(;S);+M$Ei>6-SN1vad4`F?b$H7Xd}F3~E3_3QWlz>ediMS+aksu44b-GoQ)
z_&Q)MV!KOdT4mwF(zEQ2%TJ&mUDTA{EHF4Q_NzmuL(_ro5-$?Arx)GPNe8L3<839-
zdL-WmUT$2P)9h1}+xS^HyHySzOzlc=0=?-@fxP-Kv%toX^4065!>~oPTM_!b1ZeB-
z;e;NwQvhF$YSuK1-NKDqR~;hW0_t^vs}xL3%}HHQes@Ud%o7b~feS<9!x?UK58t-5
zSC{MkX>8-n*BAU+-r&+iR~?29txo6-4RlD~c>J{JU};j)>^G(5!G7VwrHMu{Y-(iw
zsbJ`xoI<!yh*<}_3;ULuPV{`Zrl$_BMm6O9One5Hc6DL7n>nkJa=s}Cx9;kkOioxY
z9R%4f##`(GFWl~~E}$ul`~eUzbZ=NMu<51=K(I^Fdd;(|*!Khv9lLfQIJ&2LFDE&?
z)~qCzEv7b`BAX`>Uc%H}_i=W!ho#^j20V6#$7;*ba;7oQZ!n*1cd1l&a4}Tb_O0hT
zT%VKq=MOB;ZtceUZb|pcSC4UwJ5Pvze0Ak9i`Z1ZCA-|C{*PIiO2tBNT!#C4Np@;!
zlWnIr#`Pw-uM&~yvo21Luh=1OOKh@t-LC5~S*)BW^o$9e`+u%i$?RI&WS@1B|35Cd
zrc$ACKL1O#xqi7`l5w@P$(}gwD^6UMh(^D4dyLCDC%(RK{hw_a?7nk7C1)!)4n9}2
zUNMV#MaHNd+ijA)cfCqpRXQ(T9UhF=;$`!{+5eGeYej!9r=E}L9BZSJIWS$^KJgyO
z?5-{OOOAIQHs?8axl@UJa9g(NK6c34F1BG%J!c)MtSQ{n23odvO^<mjjM}j+%Vn{i
zvzzHk=YnAvhH<2GU2VgE?*HNCeekQ_9ZPqmGGVzLH}jlc#?{g`TppMA*mYM{6h6G+
z*mHOe?32E`@7>y#?e^HmQeB(eFigv0KW8rG_HO78cg@E%tYcSQJK1o1Sbbw2>9n`k
zXS*!Jb{$K*5^>NQR==~1V<GJAvS)MVK%L&PA@}M3q?w&!URWM(XL8z8WpOXtHY|@O
zuiZ6D&dzcS`fOFH^Q>6~^<Jz!n=_l3eZ*~+(09+=<%8*A_c_$L{!?YUw&<R-ENyOi
zsw8fV&JB27rkpkIno7#v9h>ewmy{XuR*ib+Bo|Dt;c^*OW7L*yOfBC|Czkr9rW1Fk
z6^QfxUoq#j8RM;@w718$4HtZ#JG%<PPF)senX`%8J*zUQ+n8KaOWQS%7oCTkaaE19
zlM@^L?#;A^yl2#O!tJfIJ?5*^1KBrK^i(n%?ipUx4|&E@1!*tFCO=F#vl;ZQ5~C7%
zV0c_owyctwwJGqQ-4nWR=C19j<X*F?)jP9!;hDvmkISf{tF~;n%cVrQ_i8F7JJ&e?
z&-8y8)?^;{R-C+d&Nk-do6YU1TC=j_q4zwNlq+_P)81ak_NsH%w5+Q36zY5c+jI#T
zYgSPmdcP>YXIrYc?t7+D1(&&&Twr~8$A|sXz0u_EqyI~jtXW;#@LC?1Q?(w~+C-vH
zG%sUQ%NbAANSU*0+v7GZmtEC{Ypt_jOSwnPJMJ1KC)cdXwp=d5WmT2!T4QdwgsADf
zJmac@lrgJi6MbViF5*^;%e97R^hBF!P4x6!&4$kD+&ndBdp(A2I#uH_*OCXeI+xYC
z*NZ;y|5NAe>e_D0uq=<|byX@aOjU+eS+QMm(&f$S*e=5`+?HW^-D?iPKcoNK9I<lK
z$vm>}nR1?6ZQJp>9NXnJOnXiF@OqZ=HLuIy^o!wIrES}$9BR+`f3`L;*}GKd3%s7D
z{hHf!+_uMDg>9LZlo79GsdsjA;q?^F*G#v^Wt!}#a<*Z(U50HKhOw4NeEQCV0i4eb
zyY;$Ws$|XT+OKD=cs`cpcs-TMg2&@ha?bSH>dadm+m>^$*LK-%(_V9KxZRe=Ww=e3
zYqf22)^=0hYx4H&5AP~Vm2f*N*>IV*+wGW!*S#v+aJj51uzF(~Ym9@wH>2|YglgW(
zYj~xMT`imGIm=~n`kZZ6Dla^aVHmdKHjGu+_PpEFJ@ZxbRBkWy=e(SL^Wyxt9JoEE
zV_6=Tr&hKlXT)omwqYA<$p_PI&r`py!R~WX&aAHOvRt<1vb<)Y&I1@Tx2ZdGGYr#X
z7^_3~o~x##w(q&*l;qtt=YnInEXy%0Z>?<XpY#8CXDUL^nYOXkneaGnRpxpfmpad`
z%C@~O!|m}JW@Tr=JrD1ETsEgCF4rn;%dY8N6<eG3b~?FXx?PT4)gH%QE!#DVGcVI}
h+a7Zb*)UAECgV)YEyE7>;W^uy8!P|j4f_LbK_s`0D?<PP

literal 0
HcmV?d00001

diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index b2426946d..c47de54ed 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -17,6 +17,7 @@ FORMAT = {
         'x-gzip': ('html-gzip.bin', 'gzip'),
         'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
         'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+        'br': ('html-br.bin', 'br')
         }
 
 class HttpCompressionTest(TestCase):
@@ -50,7 +51,8 @@ class HttpCompressionTest(TestCase):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
-        self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
+        self.assertEqual(request.headers.get('Accept-Encoding'),
+                         b'gzip,deflate,br')
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')
@@ -62,6 +64,15 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
+    def test_process_response_br(self):
+        response = self._getresponse('br')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br')
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request