diff --git a/scrapy/fetcher.py b/scrapy/fetcher.py index ee1498a34..a24192503 100644 --- a/scrapy/fetcher.py +++ b/scrapy/fetcher.py @@ -1,49 +1,28 @@ -import urlparse +from urlparse import urlparse from scrapy.spider import spiders from scrapy.http import Request from scrapy.core.manager import scrapymanager from scrapy.spider import BaseSpider -def fetch(urls, perdomain=False): - """Download a set of urls. This is starts a reactor and is suitable - for calling from a main method - not for calling from within the - framework. +def fetch(urls): + """Download the given urls and return a list of the successfully downloaded + responses. - It will return a list of Response objects for all pages successfully - downloaded. + Suitable for for calling from a script, shouldn't be called from spiders. """ - bigbucket = {} # big bucket to store response objects (per domain) - - def _add(response, domain): - if not domain in bigbucket: - bigbucket[domain] = [] - bigbucket[domain] += [response] - - # url clasification by domain - requestdata = set() - for url in urls: - domain = get_or_create_spider(url).domain_name - request = Request(url=url, callback=lambda r: _add(r, domain), dont_filter=True) - requestdata.add(request) - - scrapymanager.runonce(*requestdata) - - # returns bigbucket as dict or flatted - if perdomain: - return bigbucket - - flatbucket = [] - for domain, responses in bigbucket.iteritems(): - flatbucket.extend(responses) - return flatbucket - + map(get_or_create_spider, urls) + responses = [] + requests = [Request(url, callback=responses.append, dont_filter=True) \ + for url in urls] + scrapymanager.runonce(*requests) + return responses def get_or_create_spider(url): # XXX: hack to allow downloading pages from unknown domains spider = spiders.fromurl(url) if not spider: - domain = urlparse.urlparse(url).hostname + domain = urlparse(url).hostname spider = BaseSpider() spider.domain_name = domain spiders.add_spider(spider)