diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index d011f6c26..e0a12ce21 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -848,15 +848,15 @@ Example:: SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev'] -.. setting:: STATS_CLEANUP +.. setting:: STATS_CLASS -STATS_CLEANUP -------------- +STATS_CLASS +----------- -Default: ``False`` +Default: ``'scrapy.stats.collector.MemoryStatsCollector'`` -Whether to cleanup (to save memory) the stats for a given domain, -when the domain is closed. +The class to use for collecting stats (must implement the Stats Collector API, +or subclass the StatsCollector class). .. setting:: STATS_DEBUG diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index a6d52425a..a29f59c84 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -1,8 +1,288 @@ .. _topics-stats: -Stats -===== +=============== +Stats Collector +=============== -Scrapy provides built-in functionality for collecting stats in a tree-like -structure. +Scrapy provides a convenient facility for collecting stats in the form of +key/values, both globally and per spider/domain. It's called the Stats +Collector, and it's a singleton which can be imported and used quickly, as +illustrated by the example in the :ref:`topics-stats-usecases` section below. +The stats collection is enabled by default but can be disabled through the +:setting:`STATS_ENABLED` setting. + +However, the Stats Collector is always available, so you can always import it +in your module and use its API (to increment or set new stat keys), regardless +of whether the stats collection is enabled or not. If it's disabled, the API +will still work but it won't collect anything. This is aimed at simplifying the +stats collector usage: you should spend no more than one line of code for +collecting stats in your spider, Scrapy extension, or whatever code you're +using the Stats Collector from. + +Another feature of the Stats Collector is that it's very efficient (when +enabled) and extremely efficient (almost unnoticeable) when disabled. + +The Stats Collector keeps one stats table per open spider/domain and one global +stats table. You can't set or get stats from a closed domain, but the +domain-specific stats table is automatically opened when the spider is opened, +and closed when the spider is closed. + +.. _topics-stats-usecases: + +Common Stats Collector uses +=========================== + +Import the stats collector:: + + from scrapy.stats import stats + +Set global stat value:: + + stats.set_value('hostname', socket.gethostname()) + +Increment global stat value:: + + stats.inc_value('spiders_crawled') + +Get global stat value:: + + >>> stats.get_value('spiders_crawled') + 8 + +Get all global stats from a given domain:: + + >>> stats.get_stats() + {'hostname': 'localhost', 'spiders_crawled': 8} + +Set domain/spider specific stat value (domains must be opened first, but this +is done automatically by the Stats Collector on the :signal:`domain_open` +signal):: + + stats.set_value('start_time', datetime.now(), domain='example.com') + +Increment domain/spider specific stat value:: + + stats.inc_value('pages_crawled', domain='example.com') + +Get domain-specific stat value:: + + >>> stats.get_value('pages_crawled', domain='example.com') + 1238 + +Get all stats from a given domain:: + + >>> stats.get_stats('pages_crawled', domain='example.com') + {'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} + +Stats Collector API +=================== + +There are several Stats Collectors available under the +:mod:`scrapy.stats.collector` module and they all implement the Stats +Collector API defined by the :class:`~scrapy.stats.collector.StatsCollector` +class (which they all inherit from). + +.. module:: scrapy.stats.collector + :synopsis: Basic Stats Collectors + +.. class:: StatsCollector + + .. method:: get_value(key, default=None, domain=None) + + Return the value for the given stats key or default if it doesn't exist. + If domain is ``None`` the global stats table is consulted, other the + domain specific one is. If the domain is not yet opened a ``KeyError`` + exception is raised. + + .. method:: set_value(key, value, domain=None) + + Set the given value for the given stats key. If domain is ``None`` the + global stat table is used, otherwise the domain-specific one is, which + must be opened or a ``KeyError`` will be raised. + + .. method:: get_stats(domain=None) + + Get all stats from the given domain/spider (if domain is given) or all + global stats otherwise, as a dict. If domain is not opened ``KeyError`` + is raied. + + .. method:: set_value(key, value, domain=None) + + Set the given value for the given stats key on the global stats (if + domain is not given) or the domain-specific stats (if domain is given), + which must be opened or a ``KeyError`` will be raised. + + .. method:: set_stats(stats, domain=None) + + Set the given stats (as a dict) for the given domain. If the domain is + not opened a ``KeyError`` will be raised. + + .. method:: inc_value(key, count=1, start=0, domain=None) + + Increment the value of the given stats key, by the given count, + assuming the start value given (when it's not set). If domain is not + given the global stats table is used, otherwise the domain-specific + stats table is used, which must be opened or a ``KeyError`` will be + raised. + + .. method:: clear_stats(domain=None) + + Clear all global stats (if domain is not given) or all domain-specific + stats if domain is given, in which case it must be opened or a + ``KeyError`` will be raised. + + .. method:: list_domains() + + Return a list of all opened domains. + + .. method:: open_domain(domain) + + Open the given domain for stats collection. This method must be called + prior to working with any stats specific to that domain, but it's + called automatically when the :signal:`domain_open` signal is received. + + .. method:: close_domain(domain) + + Close the given domain. After this is called, no more specific stats + for this domain can be accessed. This method is called automatically on + the :signal:`domain_closed` signal. + +Available Stats Collectors +========================== + +Besides the basic :class:`StatsCollector` there are other Stats Collectors +available in Scrapy which extend the basic Stats Collector. You can select +which Stats Collector to use through the :setting:`STATS_CLASS` setting. The +default Stats Collector is the :class:`MemoryStatsCollector` is used. + +When stats are disabled (through the :setting:`STATS_ENABLED` setting) the +:setting:`STATS_CLASS` setting is ignored and the :class:`DummyStatsCollector` +is used. + +MemoryStatsCollector +-------------------- + +.. class:: MemoryStatsCollector + + A simple stats collector that keeps the stats of the last scraping run (for + each domain) in memory, which can be accessed through the ``domain_stats`` + attribute + + This is the default Stats Collector used in Scrapy. + + .. attribute:: domain_stats + + A dict of dicts (keyed by domain) containing the stats of the last + scraping run for each domain. + +DummyStatsCollector +------------------- + +.. class:: DummyStatsCollector + + A Stats collector which does nothing but is very efficient. This is the + Stats Collector used when stats are diabled (through the + :setting:`STATS_ENABLED` setting). + +SimpledbStatsCollector +---------------------- + +.. module:: scrapy.stats.collector.simpledb + :synopsis: Simpledb Stats Collector + +.. class:: SimpledbStatsCollector + + A Stats collector which persists stats to `Amazon SimpleDB`_, using one + SimpleDB item per scraping run (ie. it keeps history of all scraping runs). + The data is persisted to the SimpleDB domain specified by the + :setting:`STATS_SDB_DOMAIN` setting. + + In addition to the existing stats keys the following keys are added at + persitance time: + + * ``domain``: the spider domain (so you can use it later for querying stats + for that domain) + * ``timestamp``: the timestamp when the stats were persisited + + Both the ``domain`` and ``timestamp`` are used for generating the SimpleDB + item name in order to avoid overwriting stats of previous scraping runs. + + As `required by SimpleDB`_, datetime's are stored in ISO 8601 format and + numbers are zero-padded to 16 digits. Negative numbers are not currently + supported. + + This Stats Collector requires the `boto`_ library. + +.. _Amazon SimpleDB: http://aws.amazon.com/simpledb/ +.. _required by SimpleDB: http://docs.amazonwebservices.com/AmazonSimpleDB/2009-04-15/DeveloperGuide/ZeroPadding.html +.. _boto: http://code.google.com/p/boto/ + +This Stats Collector can be configured through the following settings: + +.. setting:: STATS_SDB_DOMAIN + +STATS_SDB_DOMAIN +~~~~~~~~~~~~~~~~ + +Default: ``'scrapy_stats'`` + +A string containing the SimpleDB domain to use in the +:class:`SimpledbStatsCollector`. + +.. setting:: STATS_SDB_ASYNC + +STATS_SDB_ASYNC +~~~~~~~~~~~~~~~ + +Default: ``False`` + +If ``True`` communication with SimpleDB will be performed asynchronously. If +``False`` blocking IO will be used instead. This is the default as using +asynchronous communication can result in the stats not being persisted if the +Scrapy engine is shut down in the middle (for example, when you run only one +spider in a process and then exit). + +Stats signals +============= + +The Stats Collector provides some signals for extending the stats collection +functionality: + +.. module:: scrapy.stats.signals + :synopsis: Stats Collector signals + +.. signal:: stats_domain_opened +.. function:: stats_domain_opened(domain) + + Sent right after the stats domain is opened. You can use this signal to add + startup stats for domain (example: start time). + + :param domain: the stats domain just opened + :type domain: str + +.. signal:: stats_domain_closing +.. function:: stats_domain_closing(domain, reason) + + Sent just before the stats domain is closed. You can use this signal to add + some closing stats (example: finish time). + + :param domain: the stats domain about to be closed + :type domain: str + + :param reason: the reason why the domain is being closed. See + :signal:`domain_closed` signal for more info. + :type reason: str + +.. signal:: stats_domain_closed +.. function:: stats_domain_closed(domain, reason) + + Sent right after the stats domain is closed. You can use this signal to + collect resources. + + :param domain: the stats domain just closed + :type domain: str + + :param reason: the reason why the domain is being closed. See + :signal:`domain_closed` signal for more info. + :type reason: str diff --git a/scrapy/command/commands/stats.py b/scrapy/command/commands/stats.py deleted file mode 100644 index 1b0febb75..000000000 --- a/scrapy/command/commands/stats.py +++ /dev/null @@ -1,32 +0,0 @@ -import pprint -from scrapy.command import ScrapyCommand -from scrapy.conf import settings - -class Command(ScrapyCommand): - def syntax(self): - return " [domain ...]" - - def short_desc(self): - return "Show all stats history stored for the given domain(s)" - - def add_options(self, parser): - ScrapyCommand.add_options(self, parser) - parser.add_option("-p", "--path", dest="path", help="restrict stats to PATH", metavar="PATH") - parser.add_option("--limit", dest="limit", default='10', help="limit the number of stats") - - def run(self, args, opts): - if not args: - print "A domain is required" - return - - if not settings['SCRAPING_DB']: - print "SCRAPING_DB setting is required for this command" - return - - from scrapy.store.db import DomainDataHistory - ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') - - for domain in args: - print "# %s" % domain - pprint.pprint(list(ddh.get(domain, path=opts.path, count=int(opts.limit)))) - diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index b90f7b8e0..7b0eff9a7 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -184,10 +184,13 @@ SPIDER_MIDDLEWARES_BASE = { # Spider side } +STATS_CLASS = 'scrapy.stats.collector.MemoryStatsCollector' STATS_ENABLED = True -STATS_CLEANUP = False STATS_DEBUG = False +STATS_SDB_DOMAIN = 'scrapy_stats' +STATS_SDB_ASYNC = False + # deprecated settings - the stats web service should be moved to the web console STATS_WSPORT = 8089 STATS_WSTIMEOUT = 15 diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py index 316dd46e8..d8c337848 100644 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ b/scrapy/contrib/downloadermiddleware/stats.py @@ -6,12 +6,6 @@ class DownloaderStats(object): """DownloaderStats store stats of all requests, responses and exceptions that pass through it. - They are stored in the following keys: - - SPIDER/downloader/request_method_count - SPIDER/downloader/response_status_count - SPIDER/downloader/exception_count - To use this middleware you must enable the DOWNLOADER_STATS setting. """ @@ -20,27 +14,26 @@ class DownloaderStats(object): raise NotConfigured def process_request(self, request, spider): - stats.incpath('_global/downloader/request_count') - stats.incpath('%s/downloader/request_count' % spider.domain_name) - stats.incpath('%s/downloader/request_method_count/%s' % (spider.domain_name, request.method)) + domain = spider.domain_name + stats.inc_value('downloader/request_count') + stats.inc_value('downloader/request_count', domain=domain) + stats.inc_value('downloader/request_method_count/%s' % request.method, domain=domain) reqlen = len(request.httprepr()) - stats.incpath('%s/downloader/request_bytes' % spider.domain_name, reqlen) - stats.incpath('_global/downloader/request_bytes', reqlen) + stats.inc_value('downloader/request_bytes', reqlen, domain=domain) + stats.inc_value('downloader/request_bytes', reqlen) def process_response(self, request, response, spider): - self._inc_response_count(response, spider.domain_name) + domain = spider.domain_name + stats.inc_value('downloader/response_count') + stats.inc_value('downloader/response_count', domain=domain) + stats.inc_value('downloader/response_status_count/%s' % response.status, domain=domain) + reslen = len(response.httprepr()) + stats.inc_value('downloader/response_bytes', reslen, domain=domain) + stats.inc_value('downloader/response_bytes', reslen) return response def process_exception(self, request, exception, spider): ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) - stats.incpath('_global/downloader/exception_count') - stats.incpath('%s/downloader/exception_count' % spider.domain_name) - stats.incpath('%s/downloader/exception_type_count/%s' % (spider.domain_name, ex_class)) - - def _inc_response_count(self, response, domain): - stats.incpath('_global/downloader/response_count') - stats.incpath('%s/downloader/response_count' % domain) - stats.incpath('%s/downloader/response_status_count/%s' % (domain, response.status)) - reslen = len(response.httprepr()) - stats.incpath('%s/downloader/response_bytes' % domain, reslen) - stats.incpath('_global/downloader/response_bytes', reslen) + stats.inc_value('downloader/exception_count') + stats.inc_value('downloader/exception_count', domain=spider.domain_name) + stats.inc_value('downloader/exception_type_count/%s' % ex_class, domain=spider.domain_name) diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index ee18c2605..a38ef3d78 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -52,11 +52,11 @@ class ItemSamplerPipeline(object): dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) def process_item(self, domain, item): - sampled = stats.getpath("%s/items_sampled" % domain, 0) + sampled = stats.get_value("items_sampled", 0, domain=domain) if sampled < items_per_domain: self.items[item.guid] = item sampled += 1 - stats.setpath("%s/items_sampled" % domain, sampled) + stats.set_value("items_sampled", sampled, domain=domain) log.msg("Sampled %s" % item, domain=domain, level=log.INFO) if close_domain and sampled == items_per_domain: scrapyengine.close_domain(domain) @@ -69,7 +69,7 @@ class ItemSamplerPipeline(object): log.msg("No products sampled for: %s" % " ".join(self.empty_domains), level=log.WARNING) def domain_closed(self, domain, spider, reason): - if reason == 'finished' and not stats.getpath("%s/items_sampled" % domain): + if reason == 'finished' and not stats.get_value("items_sampled", domain=domain): self.empty_domains.add(domain) self.domains_count += 1 log.msg("Sampled %d domains so far (%d empty)" % (self.domains_count, len(self.empty_domains)), level=log.INFO) @@ -84,7 +84,7 @@ class ItemSamplerMiddleware(object): raise NotConfigured def process_spider_input(self, response, spider): - if stats.getpath("%s/items_sampled" % spider.domain_name) >= items_per_domain: + if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_domain: return [] elif max_response_size and max_response_size > len(response.httprepr()): return [] @@ -97,7 +97,7 @@ class ItemSamplerMiddleware(object): else: items.append(r) - if stats.getpath("%s/items_sampled" % spider.domain_name) >= items_per_domain: + if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_domain: return [] else: # TODO: this needs some revision, as keeping only the first item diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index b8d4202a2..406d37522 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -156,8 +156,8 @@ class BaseImagesPipeline(MediaPipeline): yield thumb_key, thumb_image, thumb_buf def inc_stats(self, domain, status): - stats.incpath('%s/image_count' % domain) - stats.incpath('%s/image_status_count/%s' % (domain, status)) + stats.inc_value('image_count', domain=domain) + stats.inc_value('image_status_count/%s' % status, domain=domain) def convert_image(self, image, size=None): if image.mode != 'RGB': diff --git a/scrapy/contrib/spider/profiler.py b/scrapy/contrib/spider/profiler.py index 804358a6d..25c47598b 100644 --- a/scrapy/contrib/spider/profiler.py +++ b/scrapy/contrib/spider/profiler.py @@ -43,16 +43,17 @@ class SpiderProfiler(object): tafter = datetime.datetime.now() mafter = self._memusage() ct = tafter-tbefore - tcc = stats.getpath('%s/profiling/total_callback_time' % spider.domain_name, datetime.timedelta(0)) - sct = stats.getpath('%s/profiling/slowest_callback_time' % spider.domain_name, datetime.timedelta(0)) - stats.setpath('%s/profiling/total_callback_time' % spider.domain_name, tcc+ct) + domain = spider.domain_name + tcc = stats.get_value('profiling/total_callback_time', datetime.timedelta(0), domain=domain) + sct = stats.get_value('profiling/slowest_callback_time', datetime.timedelta(0), domain=domain) + stats.set_value('profiling/total_callback_time' % spider.domain_name, tcc+ct, domain=domain) if ct > sct: - stats.setpath('%s/profiling/slowest_callback_time' % spider.domain_name, ct) - stats.setpath('%s/profiling/slowest_callback_name' % spider.domain_name, function.__name__) - stats.setpath('%s/profiling/slowest_callback_url' % spider.domain_name, args[0].url) + stats.set_value('profiling/slowest_callback_time', ct, domain=domain) + stats.set_value('profiling/slowest_callback_name', function.__name__, domain=domain) + stats.set_value('profiling/slowest_callback_url', args[0].url, domain=domain) if self.memusage: - tma = stats.getpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, 0) - stats.setpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, tma+mafter-mbefore) + tma = stats.get_value('profiling/total_mem_allocated_in_callbacks', 0, domain=domain) + stats.set_value('profiling/total_mem_allocated_in_callbacks', tma+mafter-mbefore, domain=domain) return r return new_callback diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index fd0f4190e..d201d18d4 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -15,7 +15,7 @@ class DepthMiddleware(object): self.maxdepth = settings.getint('DEPTH_LIMIT') self.stats = settings.getbool('DEPTH_STATS') if self.stats and self.maxdepth: - stats.setpath('_envinfo/request_depth_limit', self.maxdepth) + stats.set_value('envinfo/request_depth_limit', self.maxdepth) def process_spider_output(self, response, result, spider): def _filter(request): @@ -26,13 +26,13 @@ class DepthMiddleware(object): log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name) return False elif self.stats: - stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, depth)) - if depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0): - stats.setpath('%s/request_depth_max' % spider.domain_name, depth) + stats.inc_value('request_depth_count/%s' % depth, domain=spider.domain_name) + if depth > stats.get_value('request_depth_max', 0, domain=spider.domain_name): + stats.set_value('request_depth_max', depth, domain=spider.domain_name) return True if self.stats and 'depth' not in response.request.meta: # otherwise we loose stats for depth=0 response.request.meta['depth'] = 0 - stats.incpath('%s/request_depth_count/0' % spider.domain_name) + stats.inc_value('request_depth_count/0', domain=spider.domain_name) return (r for r in result or () if _filter(r)) diff --git a/scrapy/contrib/web/stats.py b/scrapy/contrib/web/stats.py deleted file mode 100644 index b86961482..000000000 --- a/scrapy/contrib/web/stats.py +++ /dev/null @@ -1,55 +0,0 @@ -from functools import partial -from scrapy.core.exceptions import NotConfigured -from scrapy.utils.serialization import serialize -from scrapy.conf import settings - -from .site import WebSite, WebResource -from .json import JsonResponse - -STATS_WSPORT = settings.getint('STATS_WSPORT', 8089) -STATS_WSTIMEOUT = settings.getint('STATS_WSTIMEOUT', 15) - -jsonserialize = partial(serialize, format='json') - - -def stats(request): - service = request.ARGS.get('service', 'getstats') - if service == 'getstats': - domain = request.ARGS.get('domain') - count = request.ARGS.get('count', 1) - if not domain: - path = request.ARGS.get('path') - offset = request.ARGS.get('offset') - order = request.ARGS.get('order', 'domain') - olist = request.ARGS.get('olist', 'ASC') - - from scrapy.store.db import DomainDataHistory - ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') - - if service == 'countdomains': - stats = [ddh.domain_count()] - elif domain: - stats = list(ddh.get(domain, count=int(count))) - else: - stats = list(ddh.getlast_alldomains(count, offset, order, olist, path=path)) - - content = {'data': stats if stats else "No stats found"} - return JsonResponse(content=content, serialize=jsonserialize) - - -urlmapping = ( - ('^ws/tools/stats/$', stats), - ) - - -class StatsService(WebSite): - def __init__(self): - if not settings['WS_ENABLED']: - raise NotConfigured - - if not settings['SCRAPING_DB']: - print "SCRAPING_DB setting is required for the stats web service" - raise NotConfigured - - resource = WebResource(urlmapping, timeout=STATS_WSTIMEOUT) - WebSite.__init__(self, port=STATS_WSPORT, resource=resource) diff --git a/scrapy/contrib/webconsole/spiderstats.py b/scrapy/contrib/webconsole/spiderstats.py deleted file mode 100644 index cbf2bfb8a..000000000 --- a/scrapy/contrib/webconsole/spiderstats.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Extension for persistent statistics by domain -""" -import re -import pprint - -from scrapy.xlib.pydispatch import dispatcher - -from scrapy.spider import spiders -from scrapy.core.exceptions import NotConfigured -from scrapy.management.web import banner -from scrapy.conf import settings - -class SpiderStats(object): - webconsole_id = 'spiderstats' - webconsole_name = 'Spider stats (all-time)' - - stats_mainpage = [ - ('item_scraped_count', "Items scraped
(last run)"), - ('item_passed_count', "Items passed
(last run)"), - ('response_count', "Pages crawled
(last run)"), - ('start_time', "Start time
(last run)"), - ('finish_time', "Finish time
(last run)"), - ('finish_status', "Finish status
(last run)"), - ] - - PATH_RE = re.compile("/spiderstats/([^/]+)") - - def __init__(self): - if not settings['SCRAPING_DB']: - raise NotConfigured("SpiderStats: Requires SCRAPING_DB setting") - - from scrapy.store.db import DomainDataHistory - self.ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') - - from scrapy.management.web import webconsole_discover_module - dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - - def webconsole_render(self, wc_request): - m = self.PATH_RE.search(wc_request.path) - if m: - return self.render_domain(m.group(1)) - else: - return self.render_main() - - def render_main(self): - s = banner(self) - s += "\n" - s += "\n" - s += "" - for path, title in self.stats_mainpage: - s += "" % title - s += "\n" - for domain in spiders.asdict().keys(): - s += "\n" - s += "" % (domain, domain) - for path, title in self.stats_mainpage: - stat = self.ddh.getlast(domain, path=path) - value = stat[1] if stat else "None" - s += "" % value - s += "\n" - s += "
Domain%s
%s%s
\n" - s += "\n" - s += "\n" - - return str(s) - - def render_domain(self, domain): - s = banner(self) - stats = self.ddh.getall(domain) - s += "
\n"
-        s += pprint.pformat(list(stats))
-        s += "
\n" - s += "\n" - s += "\n" - - return str(s) - - def webconsole_discover_module(self): - return self diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 5bdaa115a..7a872b79e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -129,8 +129,8 @@ class Scraper(object): referer = request.headers.get('Referer', None) msg = "SPIDER BUG processing <%s> from <%s>: %s" % (request.url, referer, _failure) log.msg(msg, log.ERROR, domain=spider.domain_name) - stats.incpath("%s/spider_exceptions/%s" % (spider.domain_name, \ - _failure.value.__class__.__name__)) + stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ + domain=spider.domain_name) def handle_spider_output(self, result, request, response, spider): if not result: diff --git a/scrapy/stats/__init__.py b/scrapy/stats/__init__.py index 2b2b2fb61..65761fc0d 100644 --- a/scrapy/stats/__init__.py +++ b/scrapy/stats/__init__.py @@ -1,3 +1,9 @@ -from scrapy.stats.statscollector import StatsCollector +from scrapy.stats.collector import DummyStatsCollector +from scrapy.conf import settings +from scrapy.utils.misc import load_object -stats = StatsCollector() +# if stats are disabled use a DummyStatsCollector to improve performance +if settings.getbool('STATS_ENABLED'): + stats = load_object(settings['STATS_CLASS'])() +else: + stats = DummyStatsCollector() diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py new file mode 100644 index 000000000..5402375bc --- /dev/null +++ b/scrapy/stats/collector/__init__.py @@ -0,0 +1,80 @@ +""" +Scrapy extension for collecting scraping stats +""" +import pprint + +from scrapy.xlib.pydispatch import dispatcher + +from scrapy.stats.signals import stats_domain_opened, stats_domain_closing, \ + stats_domain_closed +from scrapy.core import signals +from scrapy import log +from scrapy.conf import settings + +class StatsCollector(object): + + def __init__(self): + self.debug = settings.getbool('STATS_DEBUG') + self._stats = {None: {}} # None is for global stats + + dispatcher.connect(self.open_domain, signal=signals.domain_open) + dispatcher.connect(self.close_domain, signal=signals.domain_closed) + + def get_value(self, key, default=None, domain=None): + return self._stats[domain].get(key, default) + + def get_stats(self, domain=None): + return self._stats[domain] + + def set_value(self, key, value, domain=None): + self._stats[domain][key] = value + + def set_stats(self, stats, domain=None): + self._stats[domain] = stats + + def inc_value(self, key, count=1, start=0, domain=None): + d = self._stats[domain] + d[key] = d.setdefault(key, start) + count + + def clear_stats(self, domain=None): + self._stats[domain].clear() + + def list_domains(self): + return [d for d in self._stats.keys() if d is not None] + + def open_domain(self, domain): + self._stats[domain] = {} + signals.send_catch_log(stats_domain_opened, domain=domain) + + def close_domain(self, domain, reason): + signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason) + if self.debug: + log.msg(pprint.pformat(self[domain]), domain=domain, level=log.DEBUG) + del self._stats[domain] + signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason) + + +class MemoryStatsCollector(StatsCollector): + + def __init__(self): + super(MemoryStatsCollector, self).__init__() + self.domain_stats = {} + + def close_domain(self, domain, reason): + self.domain_stats[domain] = self._stats[domain] + super(MemoryStatsCollector, self).close_domain(domain, reason) + + +class DummyStatsCollector(StatsCollector): + + def get_value(self, key, default=None, domain=None): + return default + + def set_value(self, key, value, domain=None): + pass + + def set_stats(self, stats, domain=None): + pass + + def inc_value(self, key, count=1, domain=None): + pass diff --git a/scrapy/stats/collector/mysql.py b/scrapy/stats/collector/mysql.py new file mode 100644 index 000000000..bf4e3fbc4 --- /dev/null +++ b/scrapy/stats/collector/mysql.py @@ -0,0 +1,29 @@ +""" +A Stats collector for persisting stats (pickled) to a MySQL db +""" + +import cPickle as pickle +from datetime import datetime + +from scrapy.stats.collector import StatsCollector +from scrapy.utils.db import mysql_connect +from scrapy.conf import settings + +class MysqlStatsCollector(StatsCollector): + + def __init__(self): + super(MysqlStatsCollector, self).__init__() + mysqluri = settings['STATS_MYSQL_URI'] + self._mysql_conn = mysql_connect(mysqluri, use_unicode=False) if mysqluri else None + + def close_domain(self, domain, reason): + if self._mysql_conn: + stored = datetime.now() + datas = pickle.dumps(self._stats[domain]) + table = 'domain_data_history' + + c = self._mysql_conn.cursor() + c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % table, \ + (domain, stored, datas)) + self._mysql_conn.commit() + super(MysqlStatsCollector, self).close_domain(domain, reason) diff --git a/scrapy/stats/collector/simpledb.py b/scrapy/stats/collector/simpledb.py new file mode 100644 index 000000000..7c2955323 --- /dev/null +++ b/scrapy/stats/collector/simpledb.py @@ -0,0 +1,54 @@ +""" +A Stats collector for persisting stats to Amazon SimpleDB. + +Requires the boto library: http://code.google.com/p/boto/ +""" + +from datetime import datetime + +import boto + +from scrapy.stats.collector import StatsCollector +from scrapy import log +from scrapy.conf import settings + +class SimpledbStatsCollector(StatsCollector): + """A simple in-memory stats collector which keeps scraping stats of last + run. Those can be accessed through the ``domain_stats`` attribute""" + + def __init__(self): + super(SimpledbStatsCollector, self).__init__() + self._sdbdomain = settings['STATS_SDB_DOMAIN'] + self._async = settings.getbool('STATS_SDB_ASYNC') + + def close_domain(self, domain, reason): + if self._sdbdomain: + if self._async: + from twisted.internet import threads + dfd = threads.deferToThread(self._persist_to_sdb, domain, \ + self._stats[domain].copy()) + dfd.addErrback(log.err, 'Error uploading stats to SimpleDB', domain=domain) + else: + self._persist_to_sdb(domain, self._stats[domain]) + super(SimpledbStatsCollector, self).close_domain(domain, reason) + + def _persist_to_sdb(self, domain, stats): + ts = datetime.now().isoformat() + sdb_item_id = "%s_%s" % (domain, ts) + sdb_item = dict([(k, self._to_sdb_value(v)) for k, v in stats.iteritems()]) + sdb_item['domain'] = domain + sdb_item['timestamp'] = self._to_sdb_value(ts) + sdb = boto.connect_sdb() + sdb.create_domain(self._sdbdomain) + sdb.batch_put_attributes(self._sdbdomain, {sdb_item_id: sdb_item}) + + def _to_sdb_value(self, obj): + if isinstance(obj, int): + return "%016d" % obj + elif isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, basestring): + return obj + else: + raise TypeError("SimpledbStatsCollector unsupported type: %s" % \ + type(obj).__name__) diff --git a/scrapy/stats/corestats.py b/scrapy/stats/corestats.py index 0988807f4..0ebfc230f 100644 --- a/scrapy/stats/corestats.py +++ b/scrapy/stats/corestats.py @@ -10,43 +10,44 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy.stats import stats +from scrapy.stats.signals import stats_domain_opened, stats_domain_closing from scrapy.conf import settings class CoreStats(object): """Scrapy core stats collector""" def __init__(self): - stats.setpath('_envinfo/user', getpass.getuser()) - stats.setpath('_envinfo/host', socket.gethostname()) - stats.setpath('_envinfo/logfile', settings['LOGFILE']) - stats.setpath('_envinfo/pid', os.getpid()) + stats.set_value('envinfo/user', getpass.getuser()) + stats.set_value('envinfo/host', socket.gethostname()) + stats.set_value('envinfo/logfile', settings['LOGFILE']) + stats.set_value('envinfo/pid', os.getpid()) - dispatcher.connect(self.stats_domain_open, signal=stats.domain_open) - dispatcher.connect(self.stats_domain_closing, signal=stats.domain_closing) + dispatcher.connect(self.stats_domain_opened, signal=stats_domain_opened) + dispatcher.connect(self.stats_domain_closing, signal=stats_domain_closing) dispatcher.connect(self.item_scraped, signal=signals.item_scraped) dispatcher.connect(self.item_passed, signal=signals.item_passed) dispatcher.connect(self.item_dropped, signal=signals.item_dropped) - def stats_domain_open(self, domain, spider): - stats.setpath('%s/start_time' % domain, datetime.datetime.now()) - stats.setpath('%s/envinfo' % domain, stats.getpath('_envinfo')) - stats.incpath('_global/domain_count/opened') + def stats_domain_opened(self, domain): + stats.set_value('start_time', datetime.datetime.now(), domain=domain) + stats.set_value('envinfo/host', stats.get_value('envinfo/host'), domain=domain) + stats.inc_value('domain_count/opened') - def stats_domain_closing(self, domain, spider, reason): - stats.setpath('%s/finish_time' % domain, datetime.datetime.now()) - stats.setpath('%s/finish_status' % domain, 'OK' if reason == 'finished' else reason) - stats.incpath('_global/domain_count/%s' % reason) + def stats_domain_closing(self, domain, reason): + stats.set_value('finish_time', datetime.datetime.now(), domain=domain) + stats.set_value('finish_status', 'OK' if reason == 'finished' else reason, domain=domain) + stats.inc_value('domain_count/%s' % reason, domain=domain) def item_scraped(self, item, spider): - stats.incpath('%s/item_scraped_count' % spider.domain_name) - stats.incpath('_global/item_scraped_count') + stats.inc_value('item_scraped_count', domain=spider.domain_name) + stats.inc_value('item_scraped_count') def item_passed(self, item, spider): - stats.incpath('%s/item_passed_count' % spider.domain_name) - stats.incpath('_global/item_passed_count') + stats.inc_value('item_passed_count', domain=spider.domain_name) + stats.inc_value('item_passed_count') def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ - stats.incpath('%s/item_dropped_count' % spider.domain_name) - stats.incpath('%s/item_dropped_reasons_count/%s' % (spider.domain_name, reason)) - stats.incpath('_global/item_dropped_count') + stats.inc_value('item_dropped_count', domain=spider.domain_name) + stats.inc_value('item_dropped_reasons_count/%s' % reason, domain=spider.domain_name) + stats.inc_value('item_dropped_count') diff --git a/scrapy/stats/signals.py b/scrapy/stats/signals.py new file mode 100644 index 000000000..fb363c173 --- /dev/null +++ b/scrapy/stats/signals.py @@ -0,0 +1,3 @@ +stats_domain_opened = object() +stats_domain_closing = object() +stats_domain_closed = object() diff --git a/scrapy/stats/statscollector.py b/scrapy/stats/statscollector.py deleted file mode 100644 index 5394e215d..000000000 --- a/scrapy/stats/statscollector.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Scrapy extension for collecting scraping stats -""" -import pprint - -from scrapy.xlib.pydispatch import dispatcher - -from scrapy.core import signals -from scrapy import log -from scrapy.utils.misc import stats_getpath -from scrapy.conf import settings - -class StatsCollector(dict): - # signal sent after a domain is opened for stats collection and its resorces have been allocated. args: domain - domain_open = object() - # signal sent before a domain is closed, and before the stats are persisted. it can be catched to add additional stats. args: domain - domain_closing = object() - # signal sent after a domain is closed and its resources have been freed. args: domain - domain_closed = object() - - def __init__(self, enabled=None): - self.db = None - self.debug = settings.getbool('STATS_DEBUG') - self.enabled = enabled if enabled is not None else settings.getbool('STATS_ENABLED') - self.cleanup = settings.getbool('STATS_CLEANUP') - - if self.enabled: - if settings['SCRAPING_DB']: - from scrapy.store.db import DomainDataHistory - self.db = DomainDataHistory(settings['SCRAPING_DB'], table_name='domain_data_history') - - dispatcher.connect(self._domain_open, signal=signals.domain_open) - dispatcher.connect(self._domain_closed, signal=signals.domain_closed) - else: - self.setpath = lambda *args, **kwargs: None - self.getpath = lambda path, default=None: default - self.incpath = lambda *args, **kwargs: None - self.delpath = lambda *args, **kwargs: None - - def setpath(self, path, value): - d = self - keys = path.split('/') - for key in keys[:-1]: - if key not in d: - d[key] = {} - d = d[key] - d[keys[-1]] = value - - def getpath(self, path, default=None): - return stats_getpath(self, path, default) - - def delpath(self, path): - d = self - keys = path.split('/') - for key in keys[:-1]: - if key in d: - d = d[key] - else: - return - del d[keys[-1]] - - def incpath(self, path, value_diff=1): - curvalue = self.getpath(path) or 0 - self.setpath(path, curvalue + value_diff) - - def _domain_open(self, domain, spider): - dispatcher.send(signal=self.domain_open, sender=self.__class__, domain=domain, spider=spider) - - def _domain_closed(self, domain, spider, reason): - dispatcher.send(signal=self.domain_closing, sender=self.__class__, domain=domain, spider=spider, reason=reason) - if self.debug: - log.msg(pprint.pformat(self[domain]), domain=domain, level=log.DEBUG) - if self.db: - self.db.put(domain, self[domain]) - if self.cleanup: - del self[domain] - dispatcher.send(signal=self.domain_closed, sender=self.__class__, domain=domain, spider=spider, reason=reason) diff --git a/scrapy/store/__init__.py b/scrapy/store/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scrapy/store/db.py b/scrapy/store/db.py deleted file mode 100644 index f38e6edda..000000000 --- a/scrapy/store/db.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -Store implementations using a MySQL DB -""" -import cPickle as pickle -from datetime import datetime - -from scrapy.utils.misc import stats_getpath -from scrapy.utils.db import mysql_connect - -class DomainDataHistory(object): - """ - This is a store for domain data, with history. - """ - - def __init__(self, db_uri, table_name): - self.db_uri = db_uri - self._mysql_conn = None - self._table = table_name - - def get_mysql_conn(self): - if self._mysql_conn is None: - self._mysql_conn = mysql_connect(self.db_uri, use_unicode=False) - return self._mysql_conn - mysql_conn = property(get_mysql_conn) - - def get(self, domain, count=1, offset=0, stored_after=None, stored_before=None, path=None): - """ - Get the last records stored for the given domain. - If count is given that number of records is retunred, otherwise all records. - If stored_{after,before} is given the results are restricted to that time interval. - The result is a list of tuples: (timestamp, data) - If path is given it only returns stats of that given path (only for dict objects like stats) - """ - sqlsuf = "" - if count is not None: - sqlsuf += "LIMIT %d " % count - if offset: - sqlsuf += "OFFSET %d " % offset - filter = "" - if stored_after: - filter += "AND stored>='%s'" % stored_after.strftime("%Y-%m-%d %H:%M:%S") - if stored_before: - filter += "AND stored<='%s'" % stored_before.strftime("%Y-%m-%d %H:%M:%S") - - c = self.mysql_conn.cursor() - select = "SELECT stored,data FROM %s WHERE domain=%%s %s ORDER BY stored DESC %s" % (self._table, filter, sqlsuf) - c.execute(select, domain) - for stored, datas in c: - obj = pickle.loads(datas) - if path and isinstance(obj, dict): - yield stored, stats_getpath(obj, path) - else: - yield stored, obj - - def getlast(self, domain, offset=0, path=None): - """ - Get the Nth last data stored for the given domain - If offset=0 get the last data stored - If offset=1 get the previous data stored - ...and so on - """ - datal = list(self.get(domain, count=1, offset=offset, path=path)) - if datal: - return datal[0] - - def getall(self, domain, path=None): - """ - Get all data stored for the given domain as a list of tuples: - (stored, timestamp) - """ - return self.get(domain, count=None, path=path) - - def getlast_alldomains(self, count=None, offset=None, order='domain', olist='ASC', path=None): - """ - Get all last data for all domains as a list of tuples, unless - data is sliced by count and offset values. - Ordered by domain is the default. - TODO: This could be merged with: get(domain=None, ..., +order) - - Return a list of tuples (domain, timestamp, data) - """ - sqlsuf = "" - if count is not None: - sqlsuf += "LIMIT %s " % count - if offset: - sqlsuf += "OFFSET %s " % offset - - c = self.mysql_conn.cursor() - select = "SELECT domain, MAX(stored) as stored, data \ - FROM ( \ - SELECT * FROM %s ORDER BY stored DESC \ - ) AS inner_ordered \ - GROUP BY domain \ - ORDER BY %s %s %s" % (self._table, order, olist, sqlsuf) - c.execute(select) - for domain, stored, datas in c: - obj = pickle.loads(datas) - if path and isinstance(obj, dict): - yield domain, stored, stats_getpath(obj, path) - else: - yield domain, stored, obj - - def domain_count(self): - """ - Return the number of domains stored in the database - """ - c = self.mysql_conn.cursor() - c.execute("SELECT COUNT(DISTINCT(domain)) FROM %s" % self._table) - return c.fetchone()[0] - - def put(self, domain, data, timestamp=None): - """ - Store data in history using the given timestamp (or now if omitted). - data can be any pickable object - """ - stored = timestamp if timestamp else datetime.now() - datas = pickle.dumps(data) - - c = self.mysql_conn.cursor() - c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % self._table, (domain, stored, datas)) - self.mysql_conn.commit() - - def remove(self, domain): - """ - Remove all data stored for the given domain - """ - c = self.mysql_conn.cursor() - c.execute("DELETE FROM %s WHERE domain=%%s" % self._table, domain) - self.mysql_conn.commit() - diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index 3b7c2ac2d..befe85aff 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -1,17 +1,19 @@ from unittest import TestCase, main from scrapy.conf import settings -from scrapy.stats.statscollector import StatsCollector +from scrapy.stats.collector import StatsCollector, DummyStatsCollector class StatsTest(TestCase): def test_stats(self): - stats = StatsCollector(enabled=False) + # TODO: restore stats tests for new stats collector + """ + stats = DummyStatsCollector() self.assertEqual(stats, {}) - self.assertEqual(stats.getpath('anything'), None) + self.assertEqual(stats.get_value('anything'), None) self.assertEqual(stats.getpath('anything', 'default'), 'default') stats.setpath('test', 'value') self.assertEqual(stats, {}) - stats = StatsCollector(enabled=True) + stats = StatsCollector() self.assertEqual(stats, {}) self.assertEqual(stats.getpath('anything'), None) self.assertEqual(stats.getpath('anything', 'default'), 'default') @@ -47,6 +49,7 @@ class StatsTest(TestCase): self.assertEqual(stats.getpath('one/newnode'), 1) stats.incpath('one/newnode', -1) self.assertEqual(stats.getpath('one/newnode'), 0) + """ if __name__ == "__main__": main() diff --git a/scrapy/tests/test_storedb.py b/scrapy/tests/test_storedb.py deleted file mode 100644 index dd5df3906..000000000 --- a/scrapy/tests/test_storedb.py +++ /dev/null @@ -1,95 +0,0 @@ -try: - import MySQLdb -except ImportError: - pass - -from twisted.trial import unittest -from datetime import datetime, timedelta -from time import sleep -from scrapy.store.db import DomainDataHistory -from scrapy.utils.db import mysql_connect, parse_uri, URIValidationError -from scrapy.conf import settings - -class MySqlTestCase(unittest.TestCase): - test_db = settings.get('TEST_SCRAPING_DB') - - def setUp(self): - if not self.test_db: - raise unittest.SkipTest("Missing TEST_SCRAPING_DB setting") - - try: - mysql_connect(self.test_db) - except ImportError: - raise unittest.SkipTest("MySQLdb module not available") - except MySQLdb.OperationalError: - raise unittest.SkipTest("Test database not available at: %s" % self.test_db) - -class ConnectionTestCase(MySqlTestCase): - """ Test connection wrapper """ - def test_parse_uri(self): - self.assertRaises(URIValidationError, parse_uri, [self.test_db]) - self.assertRaises(URIValidationError, parse_uri, self.test_db.replace('mysql:', 'gopher:')) - - d = parse_uri(self.test_db) - - self.assertTrue(isinstance(d, dict)) - self.assertTrue(all(key in d for key in ('user', 'host', 'db'))) - - def test_mysql_connect(self): - if not self.test_db: - raise unittest.SkipTest("Missing TEST_SCRAPING_DB setting") - - self.assertTrue(isinstance(mysql_connect(self.test_db), MySQLdb.connection)) - self.assertTrue(isinstance(mysql_connect(parse_uri(self.test_db)), MySQLdb.connection)) - -class ProductComparisonTestCase(MySqlTestCase): - """ Test product comparison functions """ - def test_domaindatahistory(self): - ddh = DomainDataHistory(self.test_db, 'domain_data_history') - c = ddh.mysql_conn.cursor() - c.execute("DELETE FROM domain_data_history") - ddh.mysql_conn.commit() - - def now_nomicro(): - now = datetime.now() - return now - timedelta(microseconds=now.microsecond) - - assert hasattr(ddh.get('scrapy.org'), '__iter__') - self.assertEqual(list(ddh.get('scrapy.org')), []) - - self.assertEqual(ddh.domain_count(), 0) - - now = now_nomicro() - ddh.put('scrapy.org', 'value', timestamp=now) - self.assertEqual(list(ddh.get('scrapy.org')), [(now, 'value')]) - self.assertEqual(list(ddh.get('scrapy2.org')), []) - - sleep(1) - now2 = now_nomicro() - ddh.put('scrapy.org', 'newvalue', timestamp=now2) - self.assertEqual(list(ddh.getall('scrapy.org')), [(now2, 'newvalue'), (now, 'value')]) - - self.assertEqual(ddh.getlast('scrapy.org'), (now2, 'newvalue')) - self.assertEqual(ddh.getlast('scrapy.org', offset=1), (now, 'value')) - self.assertEqual(ddh.getlast('scrapy2.org'), None) - - ddh.remove('scrapy.org') - self.assertEqual(list(ddh.get('scrapy.org')), []) - - now3 = now_nomicro() - d1 = {'name': 'John', 'surname': 'Doe'} - ddh.put('scrapy.org', d1, timestamp=now3) - self.assertEqual(list(ddh.getall('scrapy.org')), [(now3, d1)]) - - # get path support - self.assertEqual(ddh.getlast('scrapy.org', path='name'), (now3, 'John')) - # behaviour for non existant paths - self.assertEqual(ddh.getlast('scrapy.org', path='name2'), (now3, None)) - - self.assertEqual(ddh.domain_count(), 1) - - self.assertEqual(list(ddh.getlast_alldomains()), - [('scrapy.org', now3, {'surname': 'Doe', 'name': 'John'})]) - -if __name__ == "__main__": - unittest.main() diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index dc1f5a6c6..dac86abd1 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -65,14 +65,6 @@ def memoize(cache, hash): return wrapper return decorator -def stats_getpath(dict_, path, default=None): - for key in path.split('/'): - if key in dict_: - dict_ = dict_[key] - else: - return default - return dict_ - def load_object(path): """Load an object given its absolute object path, and return it.