diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index ceba04e6a..b9182223f 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -46,17 +46,19 @@ Exporter to export scraped items to different files, one per spider:: class XmlExportPipeline(object): def __init__(self): - dispatcher.connect(self.domain_opened, signals.domain_opened) - dispatcher.connect(self.domain_closed, signals.domain_closed) + dispatcher.connect(self.spider_opened, signals.spider_opened) + dispatcher.connect(self.spider_closed, signals.spider_closed) self.files = {} - def domain_opened(self, domain): + def spider_opened(self, spider): + domain = spider.domain_name file = open('%s_products.xml' % domain, 'w+b') self.files[domain] = file self.exporter = XmlItemExporter(file) self.exporter.start_exporting() - def domain_closed(self, domain): + def spider_closed(self, spider): + domain = spider.domain_name self.exporter.finish_exporting() file = self.files.pop(domain) file.close() diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index f18af83f2..5993ab68a 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -101,14 +101,14 @@ everytime a domain/spider is opened and closed:: class SpiderOpenCloseLogging(object): def __init__(self): - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - def domain_opened(self, domain, spider): - log.msg("opened domain %s" % domain) + def spider_opened(self, spider): + log.msg("opened spider %s" % spider.domain_name) - def domain_closed(self, domain, spider): - log.msg("closed domain %s" % domain) + def spider_closed(self, spider): + log.msg("closed spider %s" % spider.domain_name) .. _topics-extensions-ref-manager: diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 48aa2d00b..3fb90ce39 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -85,15 +85,15 @@ spider returns multiples items with the same id:: class DuplicatesPipeline(object): def __init__(self): - self.domaininfo = {} - dispatcher.connect(self.domain_opened, signals.domain_opened) - dispatcher.connect(self.domain_closed, signals.domain_closed) + self.duplicates = {} + dispatcher.connect(self.spider_opened, signals.spider_opened) + dispatcher.connect(self.spider_closed, signals.spider_closed) - def domain_opened(self, domain): - self.duplicates[domain] = set() + def spider_opened(self, spider): + self.duplicates[spider.domain_name] = set() - def domain_closed(self, domain): - del self.duplicates[domain] + def spider_closed(self, spider): + del self.duplicates[spider.domain_name] def process_item(self, domain, item): if item.id in self.duplicates[domain]: diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst index 9323595b5..b9214a23d 100644 --- a/docs/topics/leaks.rst +++ b/docs/topics/leaks.rst @@ -45,7 +45,7 @@ which quite often consists in answer the question: *which spider is leaking?*. The leak could also come from a custom middleware, pipeline or extension that you have written, if you are not releasing the (previously allocated) resources properly. For example, if you're allocating resources on -:signal:`domain_opened` but not releasing them on :signal:`domain_closed`. +:signal:`spider_opened` but not releasing them on :signal:`spider_closed`. .. _topics-leaks-trackrefs: diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 6faff7787..0b57b8242 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -29,68 +29,58 @@ Built-in signals reference Here's a list of signals used in Scrapy and their meaning, in alphabetical order. -domain_closed +spider_closed ------------- -.. signal:: domain_closed -.. function:: domain_closed(domain, spider, reason) +.. signal:: spider_closed +.. function:: spider_closed(spider, reason) - Sent after a spider/domain has been closed. This can be used to release - per-spider resources reserved on :signal:`domain_opened`. - - :param domain: a string which contains the domain of the spider which has - been closed - :type domain: str + Sent after a spider has been closed. This can be used to release per-spider + resources reserved on :signal:`spider_opened`. :param spider: the spider which has been closed :type spider: :class:`~scrapy.spider.BaseSpider` object - :param reason: a string which describes the reason why the domain was closed. If - it was closed because the domain has completed scraping, it the reason - is ``'finished'``. Otherwise, if the domain was manually closed by - calling the ``close_domain`` engine method, then the reason is the one + :param reason: a string which describes the reason why the spider was closed. If + it was closed because the spider has completed scraping, it the reason + is ``'finished'``. Otherwise, if the spider was manually closed by + calling the ``close_spider`` engine method, then the reason is the one passed in the ``reason`` argument of that method (which defaults to ``'cancelled'``). If the engine was shutdown (for example, by hitting Ctrl-C to stop it) the reason will be ``'shutdown'``. :type reason: str -domain_opened +spider_opened ------------- -.. signal:: domain_opened -.. function:: domain_opened(domain, spider) +.. signal:: spider_opened +.. function:: spider_opened(spider) - Sent after a spider/domain has been opened for crawling. This is typically - used to reserve per-spider resources, but can be used for any task that - needs to be performed when a spider/domain is opened. - - :param domain: a string with the domain of the spider which has been opened - :type domain: str + Sent after a spider has been opened for crawling. This is typically used to + reserve per-spider resources, but can be used for any task that needs to be + performed when a spider is opened. :param spider: the spider which has been opened :type spider: :class:`~scrapy.spider.BaseSpider` object -domain_idle +spider_idle ----------- -.. signal:: domain_idle -.. function:: domain_idle(domain, spider) +.. signal:: spider_idle +.. function:: spider_idle(spider) - Sent when a domain has gone idle, which means the spider has no further: + Sent when a spider has gone idle, which means the spider has no further: * requests waiting to be downloaded * requests scheduled * items being processed in the item pipeline If the idle state persists after all handlers of this signal have finished, - the engine starts closing the domain. After the domain has finished - closing, the :signal:`domain_closed` signal is sent. + the engine starts closing the spider. After the spider has finished + closing, the :signal:`spider_closed` signal is sent. - You can, for example, schedule some requests in your :signal:`domain_idle` - handler to prevent the domain from being closed. - - :param domain: is a string with the domain of the spider which has gone idle - :type domain: str + You can, for example, schedule some requests in your :signal:`spider_idle` + handler to prevent the spider from being closed. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spider.BaseSpider` object diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index f1c9dd095..3b3026654 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -176,7 +176,7 @@ class (which they all inherit from). Close the given domain. After this is called, no more specific stats for this domain can be accessed. This method is called automatically on - the :signal:`domain_closed` signal. + the :signal:`spider_closed` signal. Available Stats Collectors ========================== @@ -302,7 +302,7 @@ functionality: :type domain: str :param reason: the reason why the domain is being closed. See - :signal:`domain_closed` signal for more info. + :signal:`spider_closed` signal for more info. :type reason: str .. signal:: stats_domain_closed @@ -316,7 +316,7 @@ functionality: :type domain: str :param reason: the reason why the domain was closed. See - :signal:`domain_closed` signal for more info. + :signal:`spider_closed` signal for more info. :type reason: str :param domain_stats: the stats of the domain just closed. diff --git a/scrapy/contrib/closedomain.py b/scrapy/contrib/closedomain.py index 48f04bf15..1a7169030 100644 --- a/scrapy/contrib/closedomain.py +++ b/scrapy/contrib/closedomain.py @@ -23,12 +23,12 @@ class CloseDomain(object): self.tasks = {} if self.timeout: - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) if self.itempassed: dispatcher.connect(self.item_passed, signal=signals.item_passed) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - def domain_opened(self, spider): + def spider_opened(self, spider): self.tasks[spider] = reactor.callLater(self.timeout, scrapyengine.close_spider, \ spider=spider, reason='closedomain_timeout') @@ -37,7 +37,7 @@ class CloseDomain(object): if self.counts[spider] == self.itempassed: scrapyengine.close_spider(spider, 'closedomain_itempassed') - def domain_closed(self, spider): + def spider_closed(self, spider): self.counts.pop(spider, None) tsk = self.tasks.pop(spider, None) if tsk and not tsk.called: diff --git a/scrapy/contrib/delayedclosedomain.py b/scrapy/contrib/delayedclosedomain.py index ef5b68eed..6a941293c 100644 --- a/scrapy/contrib/delayedclosedomain.py +++ b/scrapy/contrib/delayedclosedomain.py @@ -21,19 +21,19 @@ class DelayedCloseDomain(object): raise NotConfigured self.opened_at = defaultdict(time) - dispatcher.connect(self.domain_idle, signal=signals.domain_idle) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_idle, signal=signals.spider_idle) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - def domain_idle(self, domain): + def spider_idle(self, spider): try: - lastseen = scrapyengine.downloader.sites[domain].lastseen + lastseen = scrapyengine.downloader.sites[spider].lastseen except KeyError: lastseen = None if not lastseen: - lastseen = self.opened_at[domain] + lastseen = self.opened_at[spider] if time() < lastseen + self.delay: raise DontCloseDomain - def domain_closed(self, domain): - self.opened_at.pop(domain, None) + def spider_closed(self, spider): + self.opened_at.pop(spider, None) diff --git a/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/contrib/downloadermiddleware/cookies.py index e64fc8b00..6569c4315 100644 --- a/scrapy/contrib/downloadermiddleware/cookies.py +++ b/scrapy/contrib/downloadermiddleware/cookies.py @@ -16,13 +16,13 @@ class CookiesMiddleware(object): def __init__(self): self.jars = defaultdict(CookieJar) - dispatcher.connect(self.domain_closed, signals.domain_closed) + dispatcher.connect(self.spider_closed, signals.spider_closed) def process_request(self, request, spider): if request.meta.get('dont_merge_cookies', False): return - jar = self.jars[spider.domain_name] + jar = self.jars[spider] cookies = self._get_request_cookies(jar, request) for cookie in cookies: jar.set_cookie_if_ok(cookie, request) @@ -37,14 +37,14 @@ class CookiesMiddleware(object): return response # extract cookies from Set-Cookie and drop invalid/expired cookies - jar = self.jars[spider.domain_name] + jar = self.jars[spider] jar.extract_cookies(response, request) self._debug_set_cookie(response) return response - def domain_closed(self, domain): - self.jars.pop(domain, None) + def spider_closed(self, spider): + self.jars.pop(spider, None) def _debug_cookie(self, request): """log Cookie header for request""" diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 04c6082ad..583a11a48 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -24,10 +24,10 @@ class HttpCacheMiddleware(object): raise NotConfigured self.cache = Cache(settings['HTTPCACHE_DIR'], sectorize=settings.getbool('HTTPCACHE_SECTORIZE')) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') - dispatcher.connect(self.open_domain, signal=signals.domain_opened) + dispatcher.connect(self.open_domain, signal=signals.spider_opened) - def open_domain(self, domain): - self.cache.open_domain(domain) + def open_domain(self, spider): + self.cache.open_domain(spider.domain_name) def process_request(self, request, spider): if not is_cacheable(request): diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 161ae9f34..eede2a1df 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -26,8 +26,8 @@ class RobotsTxtMiddleware(object): self._spider_netlocs = {} self._useragents = {} self._pending = {} - dispatcher.connect(self.domain_opened, signals.domain_opened) - dispatcher.connect(self.domain_closed, signals.domain_closed) + dispatcher.connect(self.spider_opened, signals.spider_opened) + dispatcher.connect(self.spider_closed, signals.spider_closed) def process_request(self, request, spider): useragent = self._useragents[spider] @@ -52,12 +52,12 @@ class RobotsTxtMiddleware(object): rp.parse(response.body.splitlines()) self._parsers[urlparse_cached(response).netloc] = rp - def domain_opened(self, spider): + def spider_opened(self, spider): self._spider_netlocs[spider] = set() self._useragents[spider] = getattr(spider, 'user_agent', None) \ or settings['USER_AGENT'] - def domain_closed(self, domain, spider): + def spider_closed(self, spider): for netloc in self._spider_netlocs[domain]: del self._parsers[netloc] del self._spider_netlocs[domain] diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index fc2d830ee..026b4288a 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -49,7 +49,7 @@ class ItemSamplerPipeline(object): self.items = {} self.domains_count = 0 self.empty_domains = set() - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) def process_item(self, item, spider): @@ -70,7 +70,8 @@ class ItemSamplerPipeline(object): if self.empty_domains: log.msg("No products sampled for: %s" % " ".join(self.empty_domains), level=log.WARNING) - def domain_closed(self, domain, spider, reason): + def spider_closed(self, spider, reason): + domain = spider.domain_name if reason == 'finished' and not stats.get_value("items_sampled", domain=domain): self.empty_domains.add(domain) self.domains_count += 1 diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 564001509..e5d5bbe03 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -44,10 +44,10 @@ class FSImagesStore(object): self.basedir = basedir self._mkdir(self.basedir) self.created_directories = defaultdict(set) - dispatcher.connect(self.domain_closed, signals.domain_closed) + dispatcher.connect(self.spider_closed, signals.spider_closed) - def domain_closed(self, domain): - self.created_directories.pop(domain, None) + def spider_closed(self, spider): + self.created_directories.pop(spider.domain_name, None) def persist_image(self, key, image, buf, info): absolute_path = self._get_filesystem_path(key) diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 1c1971e40..16e11bef6 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -22,14 +22,14 @@ class MediaPipeline(object): def __init__(self): self.domaininfo = {} - dispatcher.connect(self.domain_opened, signals.domain_opened) - dispatcher.connect(self.domain_closed, signals.domain_closed) + dispatcher.connect(self.spider_opened, signals.spider_opened) + dispatcher.connect(self.spider_closed, signals.spider_closed) - def domain_opened(self, spider): + def spider_opened(self, spider): self.domaininfo[spider.domain_name] = self.DomainInfo(spider) - def domain_closed(self, domain): - del self.domaininfo[domain] + def spider_closed(self, spider): + del self.domaininfo[spider.domain_name] def process_item(self, domain, item): info = self.domaininfo[domain] diff --git a/scrapy/contrib/resolver.py b/scrapy/contrib/resolver.py index b688a56c3..45222026c 100644 --- a/scrapy/contrib/resolver.py +++ b/scrapy/contrib/resolver.py @@ -16,14 +16,14 @@ class CachingResolver(object): self.resolver = _CachingThreadedResolver(reactor) reactor.installResolver(self.resolver) dispatcher.connect(self.request_received, signals.request_received) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def request_received(self, request, spider): url_hostname = urlparse_cached(request).hostname - self.spider_hostnames[spider.domain_name].add(url_hostname) + self.spider_hostnames[spider].add(url_hostname) - def domain_closed(self, spider): - for hostname in self.spider_hostnames: + def spider_closed(self, spider): + for hostname in self.spider_hostnames[spider]: self.resolver._cache.pop(hostname, None) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 2d766ddc9..6d05c5773 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -15,8 +15,8 @@ class OffsiteMiddleware(object): def __init__(self): self.host_regexes = {} - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def process_spider_output(self, response, result, spider): return (x for x in result if not isinstance(x, Request) or \ @@ -34,9 +34,9 @@ class OffsiteMiddleware(object): regex = r'^(.*\.)?(%s)$' % '|'.join(domains) return re.compile(regex) - def domain_opened(self, spider): + def spider_opened(self, spider): domains = [spider.domain_name] + spider.extra_domain_names self.host_regexes[spider] = self.get_host_regex(domains) - def domain_closed(self, spider): + def spider_closed(self, spider): del self.host_regexes[spider] diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py index 4da230a6b..3d8516801 100644 --- a/scrapy/contrib/spidermiddleware/requestlimit.py +++ b/scrapy/contrib/spidermiddleware/requestlimit.py @@ -23,44 +23,43 @@ class RequestLimitMiddleware(object): self.max_pending = {} self.dropped_count = {} - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - def domain_opened(self, domain, spider): - self.max_pending[domain] = getattr(spider, 'requests_queue_size', self.max_queue_size) - self.dropped_count[domain] = 0 + def spider_opened(self, spider): + self.max_pending[spider] = getattr(spider, 'requests_queue_size', self.max_queue_size) + self.dropped_count[spider] = 0 - def domain_closed(self, domain): - dropped_count = self.dropped_count[domain] + def spider_closed(self, spider): + dropped_count = self.dropped_count[spider] if dropped_count: - max_pending = self.max_pending[domain] + max_pending = self.max_pending[spider] log.msg('Dropped %d request(s) because the scheduler queue size limit (%d requests) was exceeded' % \ - (dropped_count, max_pending), level=log.DEBUG, domain=domain) - del self.dropped_count[domain] - del self.max_pending[domain] + (dropped_count, max_pending), level=log.DEBUG, spider=spider) + del self.dropped_count[spider] + del self.max_pending[spider] def process_spider_output(self, response, result, spider): - domain = spider.domain_name - max_pending = self.max_pending.get(domain, 0) + max_pending = self.max_pending.get(spider, 0) if max_pending: - return imap(lambda v: self._limit_requests(v, domain, max_pending), result) + return imap(lambda v: self._limit_requests(v, spider, max_pending), result) else: return result - def _limit_requests(self, request_or_other, domain, max_pending): + def _limit_requests(self, request_or_other, spider, max_pending): if isinstance(request_or_other, Request): - free_slots = max_pending - self._pending_count(domain) + free_slots = max_pending - self._pending_count(spider) if free_slots > 0: # Scheduler isn't saturated and it is fine to schedule more requests. return request_or_other else: # Skip the request and give engine time to handle other tasks. - self.dropped_count[domain] += 1 + self.dropped_count[spider] += 1 return None else: # Return others (non-requests) as is. return request_or_other - def _pending_count(self, domain): - pending = scrapyengine.scheduler.pending_requests.get(domain, []) + def _pending_count(self, spider): + pending = scrapyengine.scheduler.pending_requests.get(spider, []) return len(pending) diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 4ba7c7549..f18c98cde 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -20,49 +20,49 @@ class LiveStats(object): def __init__(self): self.domains = {} - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) dispatcher.connect(self.item_scraped, signal=signals.item_scraped) dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - def domain_opened(self, domain, spider): + def spider_opened(self, spider): pstats = SpiderStats() - self.domains[spider.domain_name] = pstats + self.domains[spider] = pstats pstats.started = datetime.now().replace(microsecond=0) pstats.finished = None - def domain_closed(self, domain, spider): - self.domains[spider.domain_name].finished = datetime.now().replace(microsecond=0) + def spider_closed(self, spider): + self.domains[spider].finished = datetime.now().replace(microsecond=0) def item_scraped(self, item, spider): - self.domains[spider.domain_name].scraped += 1 + self.domains[spider].scraped += 1 def response_downloaded(self, response, spider): # sometimes we download responses without opening/closing domains, # for example from scrapy shell - if self.domains.get(spider.domain_name): - self.domains[spider.domain_name].crawled += 1 + if self.domains.get(spider): + self.domains[spider].crawled += 1 def webconsole_render(self, wc_request): sch = scrapyengine.scheduler dwl = scrapyengine.downloader - totdomains = totscraped = totcrawled = totscheduled = totactive = totpending = totdqueued = tottransf = 0 + totdomains = totscraped = totcrawled = totscheduled = totactive = totdqueued = tottransf = 0 s = banner(self) s += "\n" s += "\n" - for d in sorted(self.domains.keys()): - scheduled = len(sch.pending_requests[d]) if d in sch.pending_requests else 0 - active = len(dwl.sites[d].active) if d in dwl.sites else 0 - dqueued = len(dwl.sites[d].queue) if d in dwl.sites else 0 - transf = len(dwl.sites[d].transferring) if d in dwl.sites else 0 - stats = self.domains[d] + for spider in sorted(self.domains.keys()): + scheduled = len(sch.pending_requests[spider]) if spider in sch.pending_requests else 0 + active = len(dwl.sites[spider].active) if spider in dwl.sites else 0 + dqueued = len(dwl.sites[spider].queue) if spider in dwl.sites else 0 + transf = len(dwl.sites[spider].transferring) if spider in dwl.sites else 0 + stats = self.domains[spider] runtime = stats.finished - stats.started if stats.finished else datetime.now() - stats.started s += '\n' % \ - (d, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(stats.finished), str(runtime)) + (spider.domain_name, stats.scraped, stats.crawled, scheduled, dqueued, active, transf, str(stats.started), str(stats.finished), str(runtime)) totdomains += 1 totscraped += stats.scraped diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 1c173fc7c..e266053ef 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -18,16 +18,16 @@ class Spiderctl(object): def __init__(self): self.running = {} self.finished = set() - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) from scrapy.management.web import webconsole_discover_module dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - def domain_opened(self, spider): + def spider_opened(self, spider): self.running[spider.domain_name] = spider - def domain_closed(self, spider): + def spider_closed(self, spider): del self.running[spider.domain_name] self.finished.add(spider.domain_name) diff --git a/scrapy/contrib_exp/pipeline/shoveitem.py b/scrapy/contrib_exp/pipeline/shoveitem.py index 584e211ba..334c07993 100644 --- a/scrapy/contrib_exp/pipeline/shoveitem.py +++ b/scrapy/contrib_exp/pipeline/shoveitem.py @@ -24,8 +24,8 @@ class ShoveItemPipeline(object): self.opts = settings['SHOVEITEM_STORE_OPT'] or {} self.stores = {} - dispatcher.connect(self.domain_opened, signal=signals.domain_opened) - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def process_item(self, domain, item): guid = str(item.guid) @@ -43,12 +43,13 @@ class ShoveItemPipeline(object): self.log(domain, item, status) return item - def domain_opened(self, domain): + def spider_opened(self, spider): + domain = spider.domain_name uri = Template(self.uritpl).substitute(domain=domain) self.stores[domain] = Shove(uri, **self.opts) - def domain_closed(self, domain): - self.stores[domain].sync() + def spider_closed(self, spider): + self.stores[spider.domain_name].sync() def log(self, domain, item, status): log.msg("Shove (%s): Item guid=%s" % (status, item.guid), level=log.DEBUG, domain=domain) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index aaaed7fd9..b54f5a2ae 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -99,7 +99,7 @@ class ExecutionEngine(object): return True def next_request(self, spider, now=False): - """Scrape the next request for the domain passed. + """Scrape the next request for the spider passed. The next request to be scraped is retrieved from the scheduler and requested from the downloader. @@ -164,7 +164,7 @@ class ExecutionEngine(object): def crawl(self, request, spider): if not request.deferred.callbacks: log.msg("Unable to crawl Request with no callback: %s" % request, - level=log.ERROR, domain=spider.domain_name) + level=log.ERROR, spider=spider) return schd = mustbe_deferred(self.schedule, request, spider) # FIXME: we can't log errors because we would be preventing them from @@ -186,7 +186,7 @@ class ExecutionEngine(object): return self.scheduler.enqueue_request(spider, request) def _mainloop(self): - """Add more domains to be scraped if the downloader has the capacity. + """Add more spiders to be scraped if the downloader has the capacity. If there is nothing else scheduled then stop the execution engine. """ @@ -198,15 +198,13 @@ class ExecutionEngine(object): return self._stop_if_idle() def download(self, request, spider): - domain = spider.domain_name - def _on_success(response): """handle the result of a page download""" assert isinstance(response, (Response, Request)) if isinstance(response, Response): response.request = request # tie request to response received log.msg(self._crawled_logline(request, response), \ - level=log.DEBUG, domain=spider.domain_name) + level=log.DEBUG, spider=spider) return response elif isinstance(response, Request): newrequest = response @@ -225,7 +223,7 @@ class ExecutionEngine(object): level = log.ERROR if errmsg: log.msg("Crawling <%s>: %s" % (request.url, errmsg), \ - level=level, domain=domain) + level=level, spider=spider) return Failure(IgnoreRequest(str(exc))) def _on_complete(_): @@ -241,38 +239,31 @@ class ExecutionEngine(object): return dwld def open_spider(self, spider): - domain = spider.domain_name - log.msg("Domain opened", domain=domain) + log.msg("Spider opened", spider=spider) self.next_request(spider) self.downloader.open_spider(spider) self.scraper.open_spider(spider) - stats.open_domain(domain) + stats.open_domain(spider.domain_name) - # XXX: sent for backwards compatibility (will be removed in Scrapy 0.8) - send_catch_log(signals.domain_open, sender=self.__class__, \ - domain=domain, spider=spider) - - send_catch_log(signals.domain_opened, sender=self.__class__, \ - domain=domain, spider=spider) + send_catch_log(signals.spider_opened, sender=self.__class__, spider=spider) def _spider_idle(self, spider): - """Called when a domain gets idle. This function is called when there + """Called when a spider gets idle. This function is called when there are no remaining pages to download or schedule. It can be called multiple times. If some extension raises a DontCloseDomain exception - (in the domain_idle signal handler) the domain is not closed until the + (in the spider_idle signal handler) the spider is not closed until the next loop and this function is guaranteed to be called (at least) once - again for this domain. + again for this spider. """ - domain = spider.domain_name try: - dispatcher.send(signal=signals.domain_idle, sender=self.__class__, \ - domain=domain, spider=spider) + dispatcher.send(signal=signals.spider_idle, sender=self.__class__, \ + spider=spider) except DontCloseDomain: self.next_request(spider) return except: - log.err("Exception catched on domain_idle signal dispatch") + log.err("Exception catched on spider_idle signal dispatch") if self.spider_is_idle(spider): self.close_spider(spider, reason='finished') @@ -283,9 +274,8 @@ class ExecutionEngine(object): def close_spider(self, spider, reason='cancelled'): """Close (cancel) spider and clear all its outstanding requests""" - domain = spider.domain_name if spider not in self.closing: - log.msg("Closing domain (%s)" % reason, domain=domain) + log.msg("Closing spider (%s)" % reason, spider=spider) self.closing[spider] = reason self.downloader.close_spider(spider) self.scheduler.clear_pending_requests(spider) @@ -298,7 +288,7 @@ class ExecutionEngine(object): return dlist def _finish_closing_spider_if_idle(self, spider): - """Call _finish_closing_spider if domain is idle""" + """Call _finish_closing_spider if spider is idle""" if self.spider_is_idle(spider) or self.killed: return self._finish_closing_spider(spider) else: @@ -310,15 +300,14 @@ class ExecutionEngine(object): def _finish_closing_spider(self, spider): """This function is called after the spider has been closed""" - domain = spider.domain_name self.scheduler.close_spider(spider) self.scraper.close_spider(spider) reason = self.closing.pop(spider, 'finished') - send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ - domain=domain, spider=spider, reason=reason) - stats.close_domain(domain, reason=reason) + send_catch_log(signal=signals.spider_closed, sender=self.__class__, \ + spider=spider, reason=reason) + stats.close_domain(spider.domain_name, reason=reason) dfd = defer.maybeDeferred(spiders.close_spider, spider) - dfd.addBoth(log.msg, "Domain closed (%s)" % reason, domain=domain) + dfd.addBoth(log.msg, "Spider closed (%s)" % reason, spider=spider) reactor.callLater(0, self._mainloop) return dfd diff --git a/scrapy/core/signals.py b/scrapy/core/signals.py index 32d6eb050..14c0abc52 100644 --- a/scrapy/core/signals.py +++ b/scrapy/core/signals.py @@ -7,9 +7,9 @@ signals here without documenting them there. engine_started = object() engine_stopped = object() -domain_opened = object() -domain_idle = object() -domain_closed = object() +spider_opened = object() +spider_idle = object() +spider_closed = object() request_received = object() request_uploaded = object() response_received = object() @@ -17,7 +17,3 @@ response_downloaded = object() item_scraped = object() item_passed = object() item_dropped = object() - -# XXX: deprecated signals (will be removed in Scrapy 0.8) -domain_open = object() - diff --git a/scrapy/log.py b/scrapy/log.py index 37018f07a..cc3db6fcd 100644 --- a/scrapy/log.py +++ b/scrapy/log.py @@ -63,26 +63,27 @@ def start(logfile=None, loglevel=None, logstdout=None): file = open(logfile, 'a') if logfile else sys.stderr log.startLogging(file, setStdout=logstdout) -def msg(message, level=INFO, component=BOT_NAME, domain=None): +def msg(message, level=INFO, component=BOT_NAME, domain=None, spider=None): """Log message according to the level""" if level > log_level: return dispatcher.send(signal=logmessage_received, message=message, level=level, \ - domain=domain) - system = domain if domain else component + domain=domain, spider=spider) + system = domain or spider.domain_name if spider else component msg_txt = unicode_to_str("%s: %s" % (level_names[level], message)) log.msg(msg_txt, system=system) -def exc(message, level=ERROR, component=BOT_NAME, domain=None): +def exc(message, level=ERROR, component=BOT_NAME, domain=None, spider=None): message = message + '\n' + format_exc() - msg(message, level, component, domain) + msg(message, level, component, domain, spider) def err(_stuff=None, _why=None, **kwargs): if ERROR > log_level: return domain = kwargs.pop('domain', None) + spider = kwargs.pop('spider', None) component = kwargs.pop('component', BOT_NAME) - kwargs['system'] = domain if domain else component + kwargs['system'] = domain or spider.domain_name if spider else component if _why: _why = unicode_to_str("ERROR: %s" % _why) log.err(_stuff, _why, **kwargs) diff --git a/scrapy/tests/test_downloadermiddleware_cookies.py b/scrapy/tests/test_downloadermiddleware_cookies.py index 39adfed55..d86429c5d 100644 --- a/scrapy/tests/test_downloadermiddleware_cookies.py +++ b/scrapy/tests/test_downloadermiddleware_cookies.py @@ -14,7 +14,7 @@ class CookiesMiddlewareTest(TestCase): self.mw = CookiesMiddleware() def tearDown(self): - self.mw.domain_closed('scrapytest.org') + self.mw.spider_closed('scrapytest.org') del self.mw def test_basic(self): diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index c4b51b0fd..f205ff264 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -89,9 +89,9 @@ class CrawlingSession(object): dispatcher.connect(self.record_signal, signals.engine_started) dispatcher.connect(self.record_signal, signals.engine_stopped) - dispatcher.connect(self.record_signal, signals.domain_opened) - dispatcher.connect(self.record_signal, signals.domain_idle) - dispatcher.connect(self.record_signal, signals.domain_closed) + dispatcher.connect(self.record_signal, signals.spider_opened) + dispatcher.connect(self.record_signal, signals.spider_idle) + dispatcher.connect(self.record_signal, signals.spider_closed) dispatcher.connect(self.item_scraped, signals.item_scraped) dispatcher.connect(self.request_received, signals.request_received) dispatcher.connect(self.response_downloaded, signals.response_downloaded) @@ -201,16 +201,16 @@ class EngineTest(unittest.TestCase): assert signals.engine_started in session.signals_catched assert signals.engine_stopped in session.signals_catched - assert signals.domain_opened in session.signals_catched - assert signals.domain_idle in session.signals_catched - assert signals.domain_closed in session.signals_catched + assert signals.spider_opened in session.signals_catched + assert signals.spider_idle in session.signals_catched + assert signals.spider_closed in session.signals_catched - self.assertEqual({'domain': session.domain, 'spider': session.spider}, - session.signals_catched[signals.domain_opened]) - self.assertEqual({'domain': session.domain, 'spider': session.spider}, - session.signals_catched[signals.domain_idle]) - self.assertEqual({'domain': session.domain, 'spider': session.spider, 'reason': 'finished'}, - session.signals_catched[signals.domain_closed]) + self.assertEqual({'spider': session.spider}, + session.signals_catched[signals.spider_opened]) + self.assertEqual({'spider': session.spider}, + session.signals_catched[signals.spider_idle]) + self.assertEqual({'spider': session.spider, 'reason': 'finished'}, + session.signals_catched[signals.spider_closed]) if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 1b533df9b..6c71213aa 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -13,7 +13,7 @@ class TestOffsiteMiddleware(TestCase): self.spider.extra_domain_names = ['scrapy.org'] self.mw = OffsiteMiddleware() - self.mw.domain_opened(self.spider) + self.mw.spider_opened(self.spider) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -28,5 +28,5 @@ class TestOffsiteMiddleware(TestCase): self.assertEquals(out, onsite_reqs) def tearDown(self): - self.mw.domain_closed(self.spider) + self.mw.spider_closed(self.spider)
DomainItems
Scraped		Pages
Crawled		Scheduler
Pending		Downloader
Queued		Downloader
Active		Downloader
Transferring		Start timeFinish timeRun time
%s%d%d%d%d%d%d%s%s%s