From efe9811d92c03b1f6f18c37fc36dc4768885225e Mon Sep 17 00:00:00 2001 From: olveyra Date: Fri, 11 Jun 2010 13:09:56 -0300 Subject: [PATCH] Populate annotation metadata with data not used by IBL extractor. --- scrapy/contrib/ibl/extraction/__init__.py | 2 +- scrapy/contrib/ibl/extraction/pageobjects.py | 4 +++- scrapy/contrib/ibl/extraction/pageparsing.py | 21 ++++++++++-------- .../samples_pageparsing.json.gz | Bin 9013 -> 9026 bytes 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/scrapy/contrib/ibl/extraction/__init__.py b/scrapy/contrib/ibl/extraction/__init__.py index 4d5cf6559..1c8fa1802 100644 --- a/scrapy/contrib/ibl/extraction/__init__.py +++ b/scrapy/contrib/ibl/extraction/__init__.py @@ -111,7 +111,7 @@ class InstanceBasedLearningExtractor(object): correctly_extracted = [c for c in correctly_extracted if \ extra_required.intersection(c.keys()) == extra_required ] if len(correctly_extracted) > 0: - return correctly_extracted, extraction_tree.template.id + return correctly_extracted, extraction_tree.template return None, None def __str__(self): diff --git a/scrapy/contrib/ibl/extraction/pageobjects.py b/scrapy/contrib/ibl/extraction/pageobjects.py index c57cdfa75..404106674 100644 --- a/scrapy/contrib/ibl/extraction/pageobjects.py +++ b/scrapy/contrib/ibl/extraction/pageobjects.py @@ -190,10 +190,11 @@ class AnnotationTag(object): for each item to be extracted from a tag attribute annotation_text - text prefix and suffix for the attribute to be extracted match_common_prefix - use this annotation for calculating across-template prefixes + metadata - dict with annotation data not used by IBL extractor """ __slots__ = ('surrounds_attribute', 'start_index', 'end_index', 'tag_attributes', 'annotation_text', 'variant_id', - 'surrounds_variant','match_common_prefix') + 'surrounds_variant','match_common_prefix', 'metadata') def __init__(self, start_index, end_index, surrounds_attribute=None, annotation_text=None, tag_attributes=None, variant_id=None, @@ -206,6 +207,7 @@ class AnnotationTag(object): self.variant_id = variant_id self.surrounds_variant = surrounds_variant self.match_common_prefix = match_common_prefix + self.metadata = {} def __str__(self): return "AnnotationTag(%s)" % ", ".join( diff --git a/scrapy/contrib/ibl/extraction/pageparsing.py b/scrapy/contrib/ibl/extraction/pageparsing.py index 486862ae5..0e82159e8 100644 --- a/scrapy/contrib/ibl/extraction/pageparsing.py +++ b/scrapy/contrib/ibl/extraction/pageparsing.py @@ -125,7 +125,7 @@ class TemplatePageParser(InstanceLearningParser): self._close_unpaired_tag() annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1) - attribute_annotations = jannotation.get('annotations', {}).items() + attribute_annotations = jannotation.pop('annotations', {}).items() for extract_attribute, tag_value in attribute_annotations: if extract_attribute == 'content': annotation.surrounds_attribute = tag_value @@ -133,10 +133,11 @@ class TemplatePageParser(InstanceLearningParser): else: annotation.tag_attributes.append((extract_attribute, tag_value)) self.annotations.append(annotation) - if jannotation.get('common_prefix', False): + if jannotation.pop('common_prefix', False): annotation.match_common_prefix = True - self.extra_required_attrs.extend(jannotation.get('required', [])) + self.extra_required_attrs.extend(jannotation.pop('required', [])) + annotation.metadata = jannotation self.next_tag_index += 1 @@ -183,7 +184,7 @@ class TemplatePageParser(InstanceLearningParser): return annotation = AnnotationTag(self.next_tag_index, None) - if jannotation.get('generated', False): + if jannotation.pop('generated', False): self.token_list.pop() annotation.start_index -= 1 if self.previous_element_class == HtmlTag: @@ -195,22 +196,24 @@ class TemplatePageParser(InstanceLearningParser): ignored = self.ignored_regions.pop() self.ignored_regions.append((ignored[0]-1, ignored[1])) - if jannotation.get('common_prefix', False): + if jannotation.pop('common_prefix', False): annotation.match_common_prefix = True - - self.extra_required_attrs.extend(jannotation.get('required', [])) + + self.extra_required_attrs.extend(jannotation.pop('required', [])) - variant_id = jannotation.get('variant', 0) + variant_id = jannotation.pop('variant', 0) if variant_id > 0: self.variant_stack.append(variant_id) annotation.surrounds_variant = variant_id - attribute_annotations = jannotation.get('annotations', {}).items() + attribute_annotations = jannotation.pop('annotations', {}).items() for extract_attribute, tag_value in attribute_annotations: if extract_attribute == 'content': annotation.surrounds_attribute = tag_value else: annotation.tag_attributes.append((extract_attribute, tag_value)) + annotation.metadata = jannotation + if annotation.annotation_text is None: self.next_tag_index += 1 if self.variant_stack: diff --git a/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz b/scrapy/tests/test_contrib_ibl/samples_pageparsing.json.gz index 20a2eb09a2f0a6918fafe602bd4328a498e3cd85..c3d950f738bc0a5747ce023d5afa4356c23aeab5 100644 GIT binary patch delta 2979 zcmV;U3taTIM#4r2ABzYG05uX!u?YMXe{^@OO>N`a@Ap?|=68K6(s}4eIyx`C7gI+Qw;G=huMXwR zjM8}Uq#KyAQ4>`h`4C}A!L`9e-2E~ zmjWFD=%B>sHZ3+7B4S%S={_*EnXgr~$!<)l6hg&xn3s?3`z&g1VOcbnssWhu3ln4$ z4`py#axbkJ=d8BS0;?GGJ2rWbWf+wdjU>am6EMJlZUN-e|JmZXd|ie2QWjvoK)*3l z?C(o+iWW5nl03Y`(mY1cd?bC=fA|&EJusFF2d!>HSYK{{=|d9-3F&TCHn}lsL6#VH z%B%fMF_eq8Mf7xg8`cn(uYt+G#r+gm$}X``%%x+GXf0OUiaH=N|7(4EM~ocj@18&j z6nS0;0=KRS9G}t9qFULbeZ}yw)_>~EKFqrR#hQFl_g7W|FUw!#d>(b)f4?+;kcPio zEo?B1Qew?Yg)8n$w1t9xL;G~c{RVt^zeIB&EUzFjq|(x&JtgIb6cHSWZx}*AHHa>8 z9mi;8W8f^gUC=)cT;ndHe7f450wa1>BH5Z^gaZXE+{A8+3DK?lV@j%mFw7D77!^tx~c z#<=fH(F|Ew7hPdzGCqk%C}C#hc*whQ9DyfbnvY;mtVXseNcN}=lAdd5h(8&|Mp+-W^AD5;L?0>PG8~f z2?FCE&|5n8VkJaQ<2{{235d#-v4=)d=)bdg2m4n1;Bt}jjF7*?C@{kiT+JLnme~$2 zJ!tu@_HFv$WdDR5e{G=J|2{*GKd)|78tqc0;;G}>s0HLnawr$i4(HD75Hwi1ZTdJy zNRs$~NcT~I95QRTG2ApUpq#fteZd-RHk5_i8hvhjm2O(z592LNY#@K-T`x^C`jJAY$uj&<6kP!VM@ z$F6ieZLSVl;tw&?<{Cftf|kyhL#q}Cd1(&6!C|<31*3KiPpSNlHU-duwmC9~4zEkO zvqW1nS~;M@PjI@PP$645kXmCZg^NyskDYVwd$`~yQ%#9l1pdu+PQ{TGR-_`i6R#MP za&%}-f6z__(pc>;1~&W|Xc^+KPrC~|^CKo5pK;csFa{^az_d}-QV+c&d)cwCf8MmAH9bIQ&BR1i<&VrADA)nHZ+jQMC?hZ4uhm2tCGXswc&nrgdJW}Z0HaVAFIHxnDkztu`*=xnwUD2L7x ze-^)oU}00t>QxD1=yR#%~6qj$z z;Ou&Bp{v{Tn^OFS2uMPQ{lc+Lp8A#Sf7CrNrUe)qYdR7o<$rMREyA&d`we=18R$hr zmm)3?r{ctx#4Ig+MmCoDGd*4z1{3Y{ z{25>70cFTgM+sRn{*F|Zj5ef7m@!oo1q;Iz10Esz(un!;HrWpeu zjG#LPUHXrL`aG3oQ|QlL1ow$Ce<)u?+goxeq6F^~VN>1$yYUQ4%1?R{tjJyCH=Y;4 zdc)Z{;N|^^-*6`@moaT?hq;%|48uIp+|8(PQidIbwk5q~;(iprqeOM3ww3gRxu`S^ z*%uk`b7g7m^+Z!L@bgSL{_`{5VfU3Ai__Cv!!)TKPVMUW99nfJGfM6He?*bjI0%-@ z(9V5R^5;J&B?X4gj2k-d3mRNdL>Je*Le~)MvtfC)8HP#b8tAxQ?l|ai-6{b%z{gY_ z51(hqzU8{TISLF#93)-w(CYhqv|K~8+n!=;t^pH55}zKx9sq23c08mDjRT=oneW&X zjvK_SZ(%fc`3g`d&4^c?fAa~^Vw!QZ)fCPz5Ain?1da$jF#3~W5P5|f^BIVPBoL8@ z=D=D!f)Rk%fJo#x5Q@p9^c>rHU?N;VA&5Srr6XPoA^ygIbKv};ai4H3!h%GPGe}Vv z0~qFTP%VCV-CchS;$#YpA|=X?ZWMib&OX(L*&(+*M&pWXZ@Ecmf1E&*jm0mh%D1l7 z4|{Y~W{?&2#$8zFQZxMt0`V>mX+u^*eHgc4fhoQD~J4Agi;XTHO|WP$eM3 z+EJSBlUN3$8ei~2f4^!?MqM2ypRGm@{hc2Z)etYzeopugh>k^a-FLUw8I}zYtT$COOQc1I0-S>v)kaus- zI|t|NL+=r?HmxYn+v7i9oiqM(AEX0BazWAPtaB9vEE6yd-`;! z(h7RL>71XRb$h+;QxPsz8*=aT_~hi>-;+xqq6mqdzB@TS_>`0NAVxP|#+A{N_gaXR zt?asX$yL3SKz}L!11lUnGNy)o^TRCt<9>`)rUyma=iiL#fw0p`z*5MG>1g z7wXmSZb4pOQdn1yi?!`ay}41%bfHFwEBtIN(v~h46C0kbZLl>hf_!a+8?|{12_`UI0~_006W~tLXp$ delta 2966 zcmV;H3u*MiMzux=ABzYG9&p4OJS@Ap?|=$w;q2J%iIw9p9cO_Z!+Xhl#I%UANv=6PM)xPvzOdU3je=t2? z3UmaZgA$+HwAf&Xh;8wt`@qy@zE;&HyD_O!2o=*|UOu+(v#7a+Wzk%!24K!FOpr}H zl)-7qy|iYWv)V!ntYXmb*yKHyVN_Byk__ukzyJff1&~kwXN%|Zbrs%AS%CQh{l-kO zPnYHtEouxTd3cGXd5oa>Ncybte=DkcU@R97THS`QzT5!Qhb9gZ(%q_Ta%0qjEHUhq zSNoY_C>L#u=;`)0tRXC41CxJ?`zf%LU1Fh_OUEA3TCBJgbwFhP*ZTC17&*@0J%JJ^ z^1KcNZe0^NKBJ*UwX#S1is50c|J0d%n05b)HTk6OudD=KmcPjPJnFoEe`)?C4S%;< z*kBl?#F~`~SKOCq3kCg#_UVrM9r*BmiRM69UO{3=rKLrCO3Du@A~+J?F@%6>5MAOr zj?v1-z*%y;pnn{=%As#=(S$~yaI(=lHm6Gv68^$RE=FRpM$;qr1T-VsLmVY@a&fU; z^9NdIUrNgTLJ*uX$&)E=f0M}ID4=*CzH^}5ItJQ4-n{994vK#q(|TVlgmwVvb>R$* zao?Gu8M3f0y28$6d=igP!pzF?+*wXXQ9H@m`F`@yeJ=f3>e=>}XvOa9*A1DooK2pNTcaP$g2eB;Do_RUYca|Bd;9Q z>h%t7O2w)D`K(fJKW}lavTMd#{wKw)COoZGD)_U;hLd$xe+=@Kt-mO9WPFHuDAW^C zsB?fC?7D!>ifsg2ZASyIW&+;G2s}tP2O2|nEfeTwBIrgO>lOLcdNYCk?e=!5-A+TI zm54+$#C(qd)n+^gY8#by77lD>0bY;bKAZ&;fM*-|?L^?MxRD3o+lj!#l*7V8Gmm?O zQkIvH%68xTe{r78?6>W(YE^9b_7Uzg&xiD2G0COF-BoKtctUIA=mr!wCU$8Rozlh- zmN1dQ?fQi^ox#Z^+I=ZgCwPf$ZAs@P!~Jb#=v^P{PR6!u=q-74Hf*1GA~^uqo`EO6 zobq4^giPJC$xFG5zx&p-kI0LWCPyT6!?eDc!ispze-XJi4K$w9dDTuF>$FLsBFbWp zUFmw-TphH;A7ZA>HGb>`EuArkRxJ+l(j0z=!*KZuM(rA&Qu!Ti3ZMgRb7T%3UYBxb ziMD36azKZl;dDKrLbh-qwZ>Em7o7qhJLlZ@aKTTeni91L{G02XiX$tmNJVldUNI)+ z=+K&=f1M1ZvD#k@Z1^+KGQ?k>b{BZ&M@%|CSjm?@r8?J@EkyNhD_XsrH%_j4?cj%qVzZYIf8 zs)WQ1if&;GNQ7rPCulwEr^$*~2gr2SxwR&}e`!N&dVtQFiHWMpUzs~lumf_(O+^XB zC{fybfwh9su!aKk(VUNrsjdcEMidV$o(0*vR2GPd3eA$+I)gNJ~X0O^4Q zg3uP>{5u^P^U=BvC61FT<7TVSS|u|z)pn)KJaMMuOpLs5CN_|NtCh;o*=!|H4xJ?| ze|`_a!lszjs}jV}@p2_lUV|itxcA#QSYPQ>4xN!#0;Rj0J1mlozk!9V)h>pigZ^q( z!zDWSQdk(HpOoVj6yMn+_~zcVPH`3q>0tK2mgkXKl=ty3;dhj^?S=IPuP$&YF5j5J z+4b5&SGVUkrT7gIkc1BVg=3pM^()z_e|um|3otg;bRec_I0o0aOOqW#^_b!3MFxx_iD1XmPwE z;xCqSD(ycq7XaB6o?-3yl|IB#0u&Y;UvFA3dTZr-RB`TJX!#ItToFlR7^t1}e_kdd zcqvW6B%?E7w(;i-o&RuhW*I??tXW190Xq4MOcVo7#fdG6Sz7vxY%KFD1uX;M3!+STznwCYY~l-l)~emxrS!9J;l~s115wdK0Sav0NC*Cct{r-2STed-?1qi zH;7x`!f5RB6`)X>5wASwe-om`G~;NiDV$#(;%_Jj91(h8^e4k0@(MNPGY|(!AR-UV zfwg!9BLJ@fk;ri%6q8BmIkxk_M7V%L5Pd{TN4yq7{EY$U!1+VtKH*q|1&JPKkfJUI zFwEhgTKw?3yZ#u&$rKnxN|YbnDEjoAeX0+$LvDMF#ueG#a+A#PyRt%{)F0gRi7mV`Pmm`8#Cj)w9Hm?98FpLC!SlcjOH1%7Q7Q&?*%`R%b)Cx-IyiNql|d0N|v}HVse9+OamDOnuE_>zNo1W!B$J-Z`&&f6bCF@*NQ>c`wI;9#8T}plY_UIiu?Hs(;B`6hM=HiieBEY{WTRjq_l4iHM?+wo(@7|tw z4$j$!-XmmfT2Y?2$A7;%XZ+_rNC$|>@!O|HhwpYYjDmgDW&7QppsUbhy_0VD^yyHg z74&-3IX^$^_Ill?B3!CAOnku`;{le-}w0i%=KAz?#wHpn4M**N2&)a#v4(RAygh)tXe z^=fywAg?betgFYx+IFSh+^A-{P$R?@ezq2AOP7m@4bRp#*qRnWzP7`@{BM(gB1Liy zo&s%s@_?G(KyT(fp`rXl{pSs5Q5eXpng$kBN*%tr{YuUTggN_Uk|Y M2M7vM%k`Q70Mci^&;S4c