mirror of https://github.com/scrapy/scrapy.git
newitem: remove ExtractorField and port googledir example
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40905
This commit is contained in:
parent
15909ef62f
commit
53983e6e23
|
|
@ -2,8 +2,8 @@
|
|||
|
||||
from scrapy.contrib_exp import newitem
|
||||
|
||||
from scrapy.contrib_exp.newitem import extractors
|
||||
from scrapy.contrib_exp import adaptors
|
||||
from scrapy.contrib_exp.newitem.extractors import ItemExtractor, adaptor
|
||||
from scrapy.contrib_exp.adaptors import extract, strip
|
||||
|
||||
|
||||
class GoogledirItem(newitem.Item):
|
||||
|
|
@ -12,9 +12,9 @@ class GoogledirItem(newitem.Item):
|
|||
description = newitem.StringField()
|
||||
|
||||
|
||||
class GoogledirItemExtractor(extractors.ItemExtractor):
|
||||
class GoogledirItemExtractor(ItemExtractor):
|
||||
item_class = GoogledirItem
|
||||
|
||||
name = extractors.ExtractorField([adaptors.extract, adaptors.strip])
|
||||
url = extractors.ExtractorField([adaptors.extract, adaptors.strip])
|
||||
description = extractors.ExtractorField([adaptors.extract, adaptors.strip])
|
||||
name = adaptor(extract, strip)
|
||||
url = adaptor(extract, strip)
|
||||
description = adaptor(extract, strip)
|
||||
|
|
|
|||
|
|
@ -38,38 +38,6 @@ class ItemExtractor(object):
|
|||
return object.__getattribute__(self, name)
|
||||
|
||||
|
||||
class ExtractorField(object):
|
||||
def __init__(self, funcs):
|
||||
if not hasattr(funcs, '__iter__'):
|
||||
raise TypeError(
|
||||
'You must initialize ExtractorField with a list of callables')
|
||||
|
||||
self._funcs = []
|
||||
|
||||
for func in funcs:
|
||||
func_args = get_func_args(func)
|
||||
takes_args = True if 'adaptor_args' in func_args else False
|
||||
self._funcs.append((func, takes_args))
|
||||
|
||||
def __call__(self, value, adaptor_args=None):
|
||||
values = [value]
|
||||
|
||||
for func, takes_args in self._funcs:
|
||||
next_round = []
|
||||
|
||||
for val in values:
|
||||
val = func(val, adaptor_args) if takes_args else func(val)
|
||||
|
||||
if isinstance(val, tuple):
|
||||
next_round.extend(val)
|
||||
elif val is not None:
|
||||
next_round.append(val)
|
||||
|
||||
values = next_round
|
||||
|
||||
return list(values)
|
||||
|
||||
|
||||
def adaptor(*funcs, **adaptor_args):
|
||||
"""A pipe adaptor implementing the tree adaption logic
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue