diff --git a/scrapy/trunk/scrapy/command/commands/parse.py b/scrapy/trunk/scrapy/command/commands/parse.py index 9b06049bd..284428fd4 100644 --- a/scrapy/trunk/scrapy/command/commands/parse.py +++ b/scrapy/trunk/scrapy/command/commands/parse.py @@ -1,46 +1,95 @@ -from scrapy.command.commands.parse_method import Command as ScrapyCommand +from scrapy.command import ScrapyCommand from scrapy.fetcher import fetch +from scrapy.http import Request +from scrapy.item import ScrapedItem from scrapy.spider import spiders +from scrapy.utils import display from scrapy import log class Command(ScrapyCommand): def syntax(self): - return "[options] " + return "[options] " def short_desc(self): - return "Parse the URL and print its results" + return "Parse the URL with the given spider method and print the results" def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--identify", dest="identify", action="store_true", help="try to use identify instead of parse") + parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show extracted links") + parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items") + parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output") + parser.add_option("--matches", dest="matches", action="store_true", help="avoid using pygments to colorize the output") + + def pipeline_process(self, item, spider, opts): + return item + + def run_method(self, spider, response, method, args, opts): + spider = spiders.fromurl(response.url) + if not spider: + log.msg('Couldnt find spider for url: %s' % response.url, level=log.ERROR) + return (), () + + items = [] + links = [] + if method: + method_fcn = method if callable(method) else getattr(spider, method, None) + if not method_fcn: + log.msg('Couldnt find method %s in %s spider' % (method, spider.domain_name)) + return (), () + + result = method_fcn(response) + links = [i for i in result if isinstance(i, Request)] + items = [self.pipeline_process(i, spider, opts) for i in result if isinstance(i, ScrapedItem)] + + return items, links + + def print_results(self, items, links, opts): + display.nocolour = opts.nocolour + if not opts.noitems: + for item in items: + for key in item.__dict__.keys(): + if key.startswith('_'): + item.__dict__.pop(key, None) + print "# Scraped Items", "-"*60 + display.pprint(list(items)) + + if not opts.nolinks: + print "# Links", "-"*68 + display.pprint(list(links)) def run(self, args, opts): - if not args: - print "A URL is required" - return + if opts.matches: + url = args[0] + method = None + else: + if len(args) < 2: + print "A URL and method is required" + return + else: + url, method = args[:2] items = set() links = set() - responses = fetch(args) - for response in responses: + for response in fetch([url]): spider = spiders.fromurl(response.url) - if spider: - if opts.identify and hasattr(spider, 'identify_products'): - ret_items, ret_links = ScrapyCommand.run_method(self, response, 'identify_products', args, opts) - else: - if hasattr(spider, 'rules'): - for rule in spider.rules: - if rule.link_extractor.match(response.url): - ret_items, ret_links = ScrapyCommand.run_method(self, response, rule.callback, args, opts) - break - else: - ret_items, ret_links = ScrapyCommand.run_method(self, response, 'parse', args, opts) - items = items.union(ret_items) - links = links.union(ret_links) - else: + if not spider: log.msg('Couldnt find spider for "%s"' % response.url) continue - if items or links: - self.print_results(items, links, opts) + if method: + ret_items, ret_links = self.run_method(spider, response, method, args, opts) + items = items.union(ret_items) + links = links.union(ret_links) + else: + if hasattr(spider, 'rules'): + for rule in spider.rules: + if rule.link_extractor.matches(response.url): + ret_items, ret_links = self.run_method(spider, response, rule.callback, args, opts) + items = items.union(ret_items) + links = links.union(ret_links) + else: + log.msg('No rules found for spider "%s", please specify a parsing method' % spider.domain_name) + continue + + self.print_results(items, links, opts) diff --git a/scrapy/trunk/scrapy/command/commands/parse_method.py b/scrapy/trunk/scrapy/command/commands/parse_method.py deleted file mode 100644 index 1f4b4585a..000000000 --- a/scrapy/trunk/scrapy/command/commands/parse_method.py +++ /dev/null @@ -1,73 +0,0 @@ -from scrapy.command import ScrapyCommand -from scrapy.fetcher import fetch -from scrapy.http import Request -from scrapy.item import ScrapedItem -from scrapy.spider import spiders -from scrapy.utils import display -from scrapy import log - -class Command(ScrapyCommand): - def syntax(self): - return "[options] " - - def short_desc(self): - return "Parse the URL with the given spider method and print the results" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("--links", dest="links", action="store_true", help="show extracted links") - parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items") - parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output") - - def pipeline_process(self, item, opts): - item.set_guid(spider) - for key in item.__dict__.keys(): - if key.startswith('_'): - item.__dict__.pop(key, None) - return item - - def run_method(self, response, method, args, opts): - spider = spiders.fromurl(response.url) - if not spider: - log.msg('Couldnt find spider for url: %s' % response.url, level=log.ERROR) - return (), () - - items = [] - links = [] - if method: - method_fcn = method if callable(method) else getattr(spider, method, None) - if not method_fcn: - log.msg('Couldnt find method %s in spider %s' % (method, spider.__name__)) - return (), () - - result = method_fcn(response) - links = [i for i in result if isinstance(i, Request)] - items = [self.pipeline_process(i, opts) for i in result if isinstance(i, ScrapedItem)] - - return items, links - - def print_results(self, items, links, opts): - display.nocolour = opts.nocolour - if not opts.noitems: - print "# Scraped Items", "-"*60 - display.pprint(list(items)) - - if opts.links: - print "# Links", "-"*68 - display.pprint(list(links)) - - def run(self, args, opts): - if len(args) < 2: - print "A URL and method is required" - return - - items = set() - links = set() - url, method = args[:2] - for response in fetch([url]): - ret_items, ret_links = self.run_method(response, method, args, opts) - items = items.union(ret_items) - links = links.union(ret_links) - - self.print_results(items, links, opts) - diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index 13ec4243d..61592820c 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -71,6 +71,11 @@ class LinkExtractor(FixedSGMLParser): if self.current_link and not self.current_link.text: self.current_link.text = data + def matches(self, url): + """This extractor matches with any url, since + it doesn't contain any patterns""" + return True + class Link(object): """ diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index dc9bb745a..646a15968 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -72,5 +72,12 @@ class RegexLinkExtractor(LinkExtractor): return links - def match(self, url): - return any(regex.search(url) for regex in self.allow_res) and not any(regex.search(url) for regex in self.deny_res) + def matches(self, url): + if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): + return False + + allowed = [regex.search(url) for regex in self.allow_res] + denied = [regex.search(url) for regex in self.deny_res] + return any(allowed) and not any(denied)