Added some missing postprocessing to items in parse command

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40455
This commit is contained in:
elpolilla 2008-12-02 02:17:15 +00:00
parent 3b7f282bbd
commit eff09f2f78
1 changed files with 4 additions and 4 deletions

View File

@ -20,6 +20,10 @@ class Command(ScrapyCommand):
parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output")
def pipeline_process(self, item, opts):
item.set_guid(spider)
for key in item.__dict__.keys():
if key.startswith('_'):
item.__dict__.pop(key, None)
return item
def run_method(self, response, method, args, opts):
@ -39,10 +43,6 @@ class Command(ScrapyCommand):
result = method_fcn(response)
links = [i for i in result if isinstance(i, Request)]
items = [self.pipeline_process(i, opts) for i in result if isinstance(i, ScrapedItem)]
for item in items:
for key in item.__dict__.keys():
if key.startswith('_'):
item.__dict__.pop(key, None)
return items, links