. Added --callback switch to crawl command

. Adding again the method 'parse_start_urls'
. Added --callback switch to parse command and removed extraction of links using rules

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40468

scrapy/trunk/scrapy/command/commands/crawl.py

@@ -2,6 +2,10 @@ from scrapy.command import ScrapyCommand
 from scrapy.core.manager import scrapymanager
 from scrapy.replay import Replay
 from scrapy.conf import settings
+from scrapy.utils.url import is_url
+from scrapy.spider import spiders
+from scrapy.http import Request
+from scrapy import log
 
 
 class Command(ScrapyCommand):
@@ -19,6 +23,7 @@ class Command(ScrapyCommand):
         parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE")
         parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR")
         parser.add_option("-n", "--nofollow", dest="nofollow", action="store_true", help="don't follow links (for use with URLs only)")
+        parser.add_option("-c", "--callback", dest="callback", action="store", help="use the provided callback for starting to crawl the given url")
 
     def process_options(self, args, opts):
         ScrapyCommand.process_options(self, args, opts)
@@ -41,4 +46,28 @@ class Command(ScrapyCommand):
             self.replay.record(args=args, opts=opts.__dict__)
 
     def run(self, args, opts):
+        if opts.callback:
+            requests = []
+            for a in args:
+                if is_url(a):
+                    spider = spiders.fromurl(a)
+                    urls = [a]
+                else:
+                    spider = spiders.fromdomain(a)
+                    urls = spider.start_urls if hasattr(spider.start_urls, '__iter__') else [spider.start_urls]
+
+                if spider:
+                    if hasattr(spider, opts.callback):
+                        requests.extend(Request(url=url, callback=getattr(spider, opts.callback)) for url in urls)
+                    else:
+                        log.msg('Callback %s doesnt exist in spider %s' % (opts.callback, spider.domain_name), log.ERROR)
+                else:
+                    log.msg('Could not found spider for %s' % a, log.ERROR)
+
+            if requests:
+                args = requests
+            else:
+                log.msg('Couldnt create any requests from the provided arguments', log.ERROR)
+                return
+
         scrapymanager.runonce(*args, **opts.__dict__)

scrapy/trunk/scrapy/command/commands/parse.py

@@ -8,17 +8,18 @@ from scrapy import log
 
 class Command(ScrapyCommand):
     def syntax(self):
-        return "[options] <url> <method>"
+        return "[options] <url>"
 
     def short_desc(self):
-        return "Parse the URL with the given spider method and print the results"
+        return "Parse the given URL and print the results"
 
     def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show extracted links")
         parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items")
         parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output")
-        parser.add_option("--matches", dest="matches", action="store_true", help="try to match and parse the url with the defined rules (if any)")
+        parser.add_option("-r", "--rules", dest="rules", action="store_true", help="try to match and parse the url with the defined rules (if any)")
+        parser.add_option("-c", "--callback", dest="callback", action="store", help="use the provided callback for parsing the url")
 
     def pipeline_process(self, item, spider, opts):
         return item
@@ -48,8 +49,8 @@ class Command(ScrapyCommand):
         if not opts.noitems:
             for item in items:
                 for key in item.__dict__.keys():
-                   if key.startswith('_'):
-                       item.__dict__.pop(key, None)
+                    if key.startswith('_'):
+                        item.__dict__.pop(key, None)
             print "# Scraped Items", "-"*60
             display.pprint(list(items))
 
@@ -58,42 +59,32 @@ class Command(ScrapyCommand):
             display.pprint(list(links))
 
     def run(self, args, opts):
-        if opts.matches:
-            url = args[0]
-            method = None
-        else:
-            if len(args) < 2:
-                print "A URL and method is required"
-                return
-            else:
-                url, method = args[:2]
+        if not args:
+            print "An URL is required"
+            return
 
-        items = []
-        links = []
-        for response in fetch([url]):
+        ret_items, ret_links = [], []
+        for response in fetch(args):
             spider = spiders.fromurl(response.url)
             if not spider:
                 log.msg('Couldnt find spider for "%s"' % response.url)
                 continue
 
-            if method:
-                ret_items, ret_links = self.run_method(spider, response, method, args, opts)
-                items.extend(ret_items)
-                links.extend(ret_links)
-            else:
-                if hasattr(spider, 'rules'):
-                    already_parsed = False
-
-                    for rule in spider.rules:
-                        links.extend(Request(url=link.url, link_text=link.text) for link in rule.link_extractor.extract_urls(response))
-                        if not already_parsed and rule.link_extractor.matches(response.url):
-                            already_parsed = True
-                            ret_items, ret_links = self.run_method(spider, response, rule.callback, args, opts)
-                            items.extend(ret_items)
-                            links.extend(ret_links)
+            if opts.callback:
+                items, links = self.run_method(spider, response, opts.callback, args, opts)
+            elif opts.rules:
+                for rule in getattr(spider, 'rules', ()):
+                    if rule.link_extractor.matches(response.url):
+                        items, links = self.run_method(spider, response, rule.callback, args, opts)
+                        break
                 else:
                     log.msg('No rules found for spider "%s", please specify a parsing method' % spider.domain_name)
                     continue
+            else:
+                items, links = self.run_method(spider, response, 'parse', args, opts)
 
-        self.print_results(items, links, opts)
+            ret_items.extend(items)
+            ret_links.extend(links)
+
+        self.print_results(ret_items, ret_links, opts)
 

scrapy/trunk/scrapy/contrib/spiders/crawl.py

@@ -2,7 +2,6 @@ import copy
 
 from scrapy.http import Request
 from scrapy.spider import BaseSpider
-from scrapy.item import ScrapedItem
 from scrapy.conf import settings
 
 class Rule(object):
@@ -61,10 +60,12 @@ class CrawlSpider(BaseSpider):
         """This function is called by the framework core for all the
         start_urls. Do not override this function, override parse_start_url
         instead."""
-        for rule in self._rules:
-            if rule.callback and rule.link_extractor.matches(response.url):
-                return self._response_downloaded(response, rule.callback, rule.cb_kwargs, follow=True)
-        return self._response_downloaded(response, None, cb_kwargs={}, follow=True)
+        return self._response_downloaded(response, self.parse_start_url, cb_kwargs={}, follow=True)
+
+    def parse_start_url(self, response):
+        """Overrideable callback function for processing start_urls. It must
+        return a list of ScrapedItems and/or Requests"""
+        return []
 
     def process_results(self, results, response):
         """This overridable method is called for each result (item or request)