diff --git a/docs/ref/spiders.rst b/docs/ref/spiders.rst index 2636c9827..e69c98bbc 100644 --- a/docs/ref/spiders.rst +++ b/docs/ref/spiders.rst @@ -39,20 +39,23 @@ method ``parse`` for each of the resulting responses. listed here. The subsequent URLs will be generated successively from data contained in the start URLs. -.. method:: BaseSpider.start_requests(urls=None) +.. method:: BaseSpider.start_requests() - A method that receives a list of URLs to scrape (for that spider) and - returns a list of Requests for those urls. + This method must return an iterable with the first Requests to crawl for + this spider. + + This is the method called by Scrapy when the spider is opened for scraping + when no particular URLs are specified. If particular URLs are specified, + the :meth:`BaseSpider.make_request_from_url` is used instead to create the + Requests. This method is also called only once from Scrapy, so it's safe to + implement it as a generator. - If urls is `None` it will use the :attr:`BaseSpider.start_urls` attribute. + The default implementation uses :meth:`BaseSpider.make_request_from_url` to + generate Requests for each url in :attr:`start_urls`. - Unless overriden, the Requests returned by this method will use the - :meth:`BaseSpider.parse` method as their callback function. - - This is also the first method called by Scrapy when it opens a spider for - scraping, so you if you want to change the Requests used to start scraping - a domain, this is the method to override. For example, if you need to start - by login in using a POST request, you could do:: + If you want to change the Requests used to start scraping a domain, this is + the method to override. For example, if you need to start by login in using + a POST request, you could do:: def start_requests(self): return [FormRequest("http://www.example.com/login", @@ -61,9 +64,19 @@ method ``parse`` for each of the resulting responses. def logged_in(self, response): # here you would extract links to follow and return Requests for - # each of them, perhaps with another callback + # each of them, with another callback pass +.. method:: BaseSpider.make_request_from_url(url) + + A method that receives a URL and returns an :class:`~scrapy.http.Request` + object to scrape that URL with this spider. This method is used to + construct the initial requests in the :meth:`start_requests` method. + + Unless overridden, this method returns Requests with the :meth:`parse` + method as their callback function, and with dont_filter parameter enabled + (see :class:`~scrapy.http.Request` class for more info). + .. method:: BaseSpider.parse(response) This is the default callback used by the :meth:`start_requests` method, and diff --git a/scrapy/contrib/prioritizers.py b/scrapy/contrib/prioritizers.py index e23a6c74a..ef68e56d5 100644 --- a/scrapy/contrib/prioritizers.py +++ b/scrapy/contrib/prioritizers.py @@ -11,7 +11,10 @@ class LessScrapedPrioritizer(object): 2. if spider was scraped before, then the less recently the spider has been scraped, the more priority it has """ - def __init__(self, elements): + def __init__(self): + # FIXME this prioritizer must be refactored + raise NotImplemented + if not settings['SCRAPING_DB']: raise NotConfigured("SCRAPING_DB setting is required") diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index dc03da567..838749477 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -28,13 +28,8 @@ class ExecutionManager(object): log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys())) scheduler = load_object(settings['SCHEDULER'])() - scrapyengine.configure(scheduler=scheduler) - - self.prioritizer_class = load_object(settings['PRIORITIZER']) - - requests = self._parse_args(args) - self.priorities = self.prioritizer_class(requests.keys()) + self.domainprio = load_object(settings['PRIORITIZER'])() def crawl(self, *args): """Schedule the given args for crawling. args is a list of urls or domains""" @@ -43,7 +38,7 @@ class ExecutionManager(object): # schedule initial requets to be scraped at engine start for domain in requests or (): spider = spiders.fromdomain(domain) - priority = self.priorities.get_priority(domain) + priority = self.domainprio.get_priority(domain) for request in requests[domain]: scrapyengine.crawl(request, spider, domain_priority=priority) @@ -71,14 +66,10 @@ class ExecutionManager(object): signal.signal(signal.SIGBREAK, signal.SIG_IGN) def reload_spiders(self): - """ - Reload all enabled spiders except for the ones that are currently + """Reload all enabled spiders except for the ones that are currently running. - """ spiders.reload(skip_domains=scrapyengine.open_domains) - # reload priorities for the new domains - self.priorities = self.prioritizer_class(spiders.asdict(include_disabled=False).keys()) def _install_signals(self): def sig_handler_terminate(signalinfo, param): @@ -122,8 +113,8 @@ class ExecutionManager(object): for url in urls: spider = spiders.fromurl(url) if spider: - reqs = spider.start_requests([url]) - perdomain.setdefault(spider.domain_name, []).extend(reqs) + req = spider.make_request_from_url(url) + perdomain.setdefault(spider.domain_name, []).append(req) else: log.msg('Could not find spider for <%s>' % url, log.ERROR) diff --git a/scrapy/core/prioritizers.py b/scrapy/core/prioritizers.py index 214a475a3..26424b70e 100644 --- a/scrapy/core/prioritizers.py +++ b/scrapy/core/prioritizers.py @@ -17,9 +17,6 @@ class NullPrioritizer(object): """ This prioritizer always return the same priority (1) """ - def __init__(self, elements): - pass - def get_priority(self, element): return 1 @@ -27,19 +24,6 @@ class RandomPrioritizer(object): """ This prioritizer always return a random priority """ - def __init__(self, elements): - self.count = len(elements) - def get_priority(self, element): - return random.randrange(0, self.count) + return random.randrange(0, 1000) -class AlphabeticPrioritizer(object): - """ - This prioritizer priotizes based on the alphabetic order of the element - """ - def __init__(self, elements): - self.elements = elements[:] - self.elements.sort() - - def get_priority(self, element): - return self.elements.index(element) diff --git a/scrapy/spider/models.py b/scrapy/spider/models.py index f22a297bd..33bf3427d 100644 --- a/scrapy/spider/models.py +++ b/scrapy/spider/models.py @@ -63,32 +63,11 @@ class BaseSpider(object): """ log.msg(message, domain=self.domain_name, level=level) - def start_requests(self, urls=None): - """Return the requests to crawl when this spider is opened for - scraping. urls contain the urls passed from command line (if any), - otherwise None if the entire domain was requested for scraping. + def start_requests(self): + return [self.make_request_from_url(url) for url in self.start_urls] - This function must return a list of Requests to be crawled, based on - the given urls. The Requests must include a callback function which - must return a list of: - - * Request's for further crawling - * ScrapedItem's for processing - * Both - - Or None (which will be treated the same way as an empty list) - - When a Request object is returned, the Request is scheduled, then - downloaded and finally its results is handled to the Request callback. - - When a ScrapedItem is returned, it is passed to the item pipeline. - - Unless this method is overrided, the start_urls attribute will be used - to create the initial requests (when urls is None). - """ - if urls is None: - urls = self.start_urls - return [Request(url, callback=self.parse, dont_filter=True) for url in urls] + def make_request_from_url(self, url): + return Request(url, callback=self.parse, dont_filter=True) def parse(self, response): """This is the default callback function used to parse the start