Simplified domain prioritizers, so that they don't receive domains in the

constructor (domain prioritizers will be refactored later anyway) and
simplified Scrapy Manager code thanks to this.

Added make_request_from_url method to BaseSpider, splitting funtionality to
create requests from URLs which was previously done all in start_requests.

docs/ref/spiders.rst

@@ -39,20 +39,23 @@ method ``parse`` for each of the resulting responses.
     listed here. The subsequent URLs will be generated successively from data
     contained in the start URLs.
 
-.. method:: BaseSpider.start_requests(urls=None)
+.. method:: BaseSpider.start_requests()
 
-    A method that receives a list of URLs to scrape (for that spider) and
-    returns a list of Requests for those urls.
+    This method must return an iterable with the first Requests to crawl for
+    this spider. 
+    
+    This is the method called by Scrapy when the spider is opened for scraping
+    when no particular URLs are specified. If particular URLs are specified,
+    the :meth:`BaseSpider.make_request_from_url` is used instead to create the
+    Requests. This method is also called only once from Scrapy, so it's safe to
+    implement it as a generator.
 
-    If urls is `None` it will use the :attr:`BaseSpider.start_urls` attribute.
+    The default implementation uses :meth:`BaseSpider.make_request_from_url` to
+    generate Requests for each url in :attr:`start_urls`.
 
-    Unless overriden, the Requests returned by this method will use the
-    :meth:`BaseSpider.parse` method as their callback function.
-
-    This is also the first method called by Scrapy when it opens a spider for
-    scraping, so you if you want to change the Requests used to start scraping
-    a domain, this is the method to override. For example, if you need to start
-    by login in using a POST request, you could do::
+    If you want to change the Requests used to start scraping a domain, this is
+    the method to override. For example, if you need to start by login in using
+    a POST request, you could do::
 
         def start_requests(self):
             return [FormRequest("http://www.example.com/login", 
@@ -61,9 +64,19 @@ method ``parse`` for each of the resulting responses.
 
         def logged_in(self, response):
             # here you would extract links to follow and return Requests for
-            # each of them, perhaps with another callback
+            # each of them, with another callback
             pass
 
+.. method:: BaseSpider.make_request_from_url(url)
+
+    A method that receives a URL and returns an :class:`~scrapy.http.Request`
+    object to scrape that URL with this spider. This method is used to
+    construct the initial requests in the :meth:`start_requests` method.
+
+    Unless overridden, this method returns Requests with the :meth:`parse`
+    method as their callback function, and with dont_filter parameter enabled
+    (see :class:`~scrapy.http.Request` class for more info).
+
 .. method:: BaseSpider.parse(response)
 
     This is the default callback used by the :meth:`start_requests` method, and

scrapy/contrib/prioritizers.py

@@ -11,7 +11,10 @@ class LessScrapedPrioritizer(object):
     2. if spider was scraped before, then the less recently the spider
        has been scraped, the more priority it has
     """
-    def __init__(self, elements):
+    def __init__(self):
+        # FIXME this prioritizer must be refactored
+        raise NotImplemented
+
         if not settings['SCRAPING_DB']:
             raise NotConfigured("SCRAPING_DB setting is required")
 

scrapy/core/manager.py

@@ -28,13 +28,8 @@ class ExecutionManager(object):
         log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()))
 
         scheduler = load_object(settings['SCHEDULER'])()
-
         scrapyengine.configure(scheduler=scheduler)
-
-        self.prioritizer_class = load_object(settings['PRIORITIZER'])
-
-        requests = self._parse_args(args)
-        self.priorities = self.prioritizer_class(requests.keys())
+        self.domainprio = load_object(settings['PRIORITIZER'])()
         
     def crawl(self, *args):
         """Schedule the given args for crawling. args is a list of urls or domains"""
@@ -43,7 +38,7 @@ class ExecutionManager(object):
         # schedule initial requets to be scraped at engine start
         for domain in requests or ():
             spider = spiders.fromdomain(domain) 
-            priority = self.priorities.get_priority(domain) 
+            priority = self.domainprio.get_priority(domain)
             for request in requests[domain]:
                 scrapyengine.crawl(request, spider, domain_priority=priority)
 
@@ -71,14 +66,10 @@ class ExecutionManager(object):
             signal.signal(signal.SIGBREAK, signal.SIG_IGN)
 
     def reload_spiders(self):
-        """
-        Reload all enabled spiders except for the ones that are currently
+        """Reload all enabled spiders except for the ones that are currently
         running.
-
         """
         spiders.reload(skip_domains=scrapyengine.open_domains)
-        # reload priorities for the new domains
-        self.priorities = self.prioritizer_class(spiders.asdict(include_disabled=False).keys())
 
     def _install_signals(self):
         def sig_handler_terminate(signalinfo, param):
@@ -122,8 +113,8 @@ class ExecutionManager(object):
         for url in urls:
             spider = spiders.fromurl(url)
             if spider:
-                reqs = spider.start_requests([url])
-                perdomain.setdefault(spider.domain_name, []).extend(reqs)
+                req = spider.make_request_from_url(url)
+                perdomain.setdefault(spider.domain_name, []).append(req)
             else:
                 log.msg('Could not find spider for <%s>' % url, log.ERROR)
 

scrapy/core/prioritizers.py

@@ -17,9 +17,6 @@ class NullPrioritizer(object):
     """
     This prioritizer always return the same priority (1)
     """
-    def __init__(self, elements):
-        pass
-
     def get_priority(self, element):
         return 1
 
@@ -27,19 +24,6 @@ class RandomPrioritizer(object):
     """
     This prioritizer always return a random priority
     """
-    def __init__(self, elements):
-        self.count = len(elements)
-
     def get_priority(self, element):
-        return random.randrange(0, self.count)
+        return random.randrange(0, 1000)
 
-class AlphabeticPrioritizer(object):
-    """
-    This prioritizer priotizes based on the alphabetic order of the element
-    """
-    def __init__(self, elements):
-        self.elements = elements[:]
-        self.elements.sort()
-
-    def get_priority(self, element):
-        return self.elements.index(element)

scrapy/spider/models.py

@@ -63,32 +63,11 @@ class BaseSpider(object):
         """
         log.msg(message, domain=self.domain_name, level=level)
 
-    def start_requests(self, urls=None):
-        """Return the requests to crawl when this spider is opened for
-        scraping. urls contain the urls passed from command line (if any),
-        otherwise None if the entire domain was requested for scraping.
+    def start_requests(self):
+        return [self.make_request_from_url(url) for url in self.start_urls]
 
-        This function must return a list of Requests to be crawled, based on
-        the given urls. The Requests must include a callback function which
-        must return a list of:
-
-        * Request's for further crawling
-        * ScrapedItem's for processing
-        * Both
-
-        Or None (which will be treated the same way as an empty list)
-
-        When a Request object is returned, the Request is scheduled, then
-        downloaded and finally its results is handled to the Request callback.
-
-        When a ScrapedItem is returned, it is passed to the item pipeline.
-
-        Unless this method is overrided, the start_urls attribute will be used
-        to create the initial requests (when urls is None).
-        """
-        if urls is None:
-            urls = self.start_urls
-        return [Request(url, callback=self.parse, dont_filter=True) for url in urls]
+    def make_request_from_url(self, url):
+        return Request(url, callback=self.parse, dont_filter=True)
 
     def parse(self, response):
         """This is the default callback function used to parse the start