mirror of https://github.com/scrapy/scrapy.git
Complete stats refactoring and documentation
--HG-- rename : scrapy/stats/statscollector.py => scrapy/stats/collector/__init__.py
This commit is contained in:
Pablo Hoffman
parent
0227c53b13
commit
8480f49cf6
|
|
@ -848,15 +848,15 @@ Example::
|
|||
|
||||
SPIDER_MODULES = ['mybot.spiders_prod', 'mybot.spiders_dev']
|
||||
|
||||
.. setting:: STATS_CLEANUP
|
||||
.. setting:: STATS_CLASS
|
||||
|
||||
STATS_CLEANUP
|
||||
-------------
|
||||
STATS_CLASS
|
||||
-----------
|
||||
|
||||
Default: ``False``
|
||||
Default: ``'scrapy.stats.collector.MemoryStatsCollector'``
|
||||
|
||||
Whether to cleanup (to save memory) the stats for a given domain,
|
||||
when the domain is closed.
|
||||
The class to use for collecting stats (must implement the Stats Collector API,
|
||||
or subclass the StatsCollector class).
|
||||
|
||||
.. setting:: STATS_DEBUG
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,288 @@
|
|||
.. _topics-stats:
|
||||
|
||||
Stats
|
||||
=====
|
||||
===============
|
||||
Stats Collector
|
||||
===============
|
||||
|
||||
Scrapy provides built-in functionality for collecting stats in a tree-like
|
||||
structure.
|
||||
Scrapy provides a convenient facility for collecting stats in the form of
|
||||
key/values, both globally and per spider/domain. It's called the Stats
|
||||
Collector, and it's a singleton which can be imported and used quickly, as
|
||||
illustrated by the example in the :ref:`topics-stats-usecases` section below.
|
||||
|
||||
The stats collection is enabled by default but can be disabled through the
|
||||
:setting:`STATS_ENABLED` setting.
|
||||
|
||||
However, the Stats Collector is always available, so you can always import it
|
||||
in your module and use its API (to increment or set new stat keys), regardless
|
||||
of whether the stats collection is enabled or not. If it's disabled, the API
|
||||
will still work but it won't collect anything. This is aimed at simplifying the
|
||||
stats collector usage: you should spend no more than one line of code for
|
||||
collecting stats in your spider, Scrapy extension, or whatever code you're
|
||||
using the Stats Collector from.
|
||||
|
||||
Another feature of the Stats Collector is that it's very efficient (when
|
||||
enabled) and extremely efficient (almost unnoticeable) when disabled.
|
||||
|
||||
The Stats Collector keeps one stats table per open spider/domain and one global
|
||||
stats table. You can't set or get stats from a closed domain, but the
|
||||
domain-specific stats table is automatically opened when the spider is opened,
|
||||
and closed when the spider is closed.
|
||||
|
||||
.. _topics-stats-usecases:
|
||||
|
||||
Common Stats Collector uses
|
||||
===========================
|
||||
|
||||
Import the stats collector::
|
||||
|
||||
from scrapy.stats import stats
|
||||
|
||||
Set global stat value::
|
||||
|
||||
stats.set_value('hostname', socket.gethostname())
|
||||
|
||||
Increment global stat value::
|
||||
|
||||
stats.inc_value('spiders_crawled')
|
||||
|
||||
Get global stat value::
|
||||
|
||||
>>> stats.get_value('spiders_crawled')
|
||||
8
|
||||
|
||||
Get all global stats from a given domain::
|
||||
|
||||
>>> stats.get_stats()
|
||||
{'hostname': 'localhost', 'spiders_crawled': 8}
|
||||
|
||||
Set domain/spider specific stat value (domains must be opened first, but this
|
||||
is done automatically by the Stats Collector on the :signal:`domain_open`
|
||||
signal)::
|
||||
|
||||
stats.set_value('start_time', datetime.now(), domain='example.com')
|
||||
|
||||
Increment domain/spider specific stat value::
|
||||
|
||||
stats.inc_value('pages_crawled', domain='example.com')
|
||||
|
||||
Get domain-specific stat value::
|
||||
|
||||
>>> stats.get_value('pages_crawled', domain='example.com')
|
||||
1238
|
||||
|
||||
Get all stats from a given domain::
|
||||
|
||||
>>> stats.get_stats('pages_crawled', domain='example.com')
|
||||
{'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
|
||||
|
||||
Stats Collector API
|
||||
===================
|
||||
|
||||
There are several Stats Collectors available under the
|
||||
:mod:`scrapy.stats.collector` module and they all implement the Stats
|
||||
Collector API defined by the :class:`~scrapy.stats.collector.StatsCollector`
|
||||
class (which they all inherit from).
|
||||
|
||||
.. module:: scrapy.stats.collector
|
||||
:synopsis: Basic Stats Collectors
|
||||
|
||||
.. class:: StatsCollector
|
||||
|
||||
.. method:: get_value(key, default=None, domain=None)
|
||||
|
||||
Return the value for the given stats key or default if it doesn't exist.
|
||||
If domain is ``None`` the global stats table is consulted, other the
|
||||
domain specific one is. If the domain is not yet opened a ``KeyError``
|
||||
exception is raised.
|
||||
|
||||
.. method:: set_value(key, value, domain=None)
|
||||
|
||||
Set the given value for the given stats key. If domain is ``None`` the
|
||||
global stat table is used, otherwise the domain-specific one is, which
|
||||
must be opened or a ``KeyError`` will be raised.
|
||||
|
||||
.. method:: get_stats(domain=None)
|
||||
|
||||
Get all stats from the given domain/spider (if domain is given) or all
|
||||
global stats otherwise, as a dict. If domain is not opened ``KeyError``
|
||||
is raied.
|
||||
|
||||
.. method:: set_value(key, value, domain=None)
|
||||
|
||||
Set the given value for the given stats key on the global stats (if
|
||||
domain is not given) or the domain-specific stats (if domain is given),
|
||||
which must be opened or a ``KeyError`` will be raised.
|
||||
|
||||
.. method:: set_stats(stats, domain=None)
|
||||
|
||||
Set the given stats (as a dict) for the given domain. If the domain is
|
||||
not opened a ``KeyError`` will be raised.
|
||||
|
||||
.. method:: inc_value(key, count=1, start=0, domain=None)
|
||||
|
||||
Increment the value of the given stats key, by the given count,
|
||||
assuming the start value given (when it's not set). If domain is not
|
||||
given the global stats table is used, otherwise the domain-specific
|
||||
stats table is used, which must be opened or a ``KeyError`` will be
|
||||
raised.
|
||||
|
||||
.. method:: clear_stats(domain=None)
|
||||
|
||||
Clear all global stats (if domain is not given) or all domain-specific
|
||||
stats if domain is given, in which case it must be opened or a
|
||||
``KeyError`` will be raised.
|
||||
|
||||
.. method:: list_domains()
|
||||
|
||||
Return a list of all opened domains.
|
||||
|
||||
.. method:: open_domain(domain)
|
||||
|
||||
Open the given domain for stats collection. This method must be called
|
||||
prior to working with any stats specific to that domain, but it's
|
||||
called automatically when the :signal:`domain_open` signal is received.
|
||||
|
||||
.. method:: close_domain(domain)
|
||||
|
||||
Close the given domain. After this is called, no more specific stats
|
||||
for this domain can be accessed. This method is called automatically on
|
||||
the :signal:`domain_closed` signal.
|
||||
|
||||
Available Stats Collectors
|
||||
==========================
|
||||
|
||||
Besides the basic :class:`StatsCollector` there are other Stats Collectors
|
||||
available in Scrapy which extend the basic Stats Collector. You can select
|
||||
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
|
||||
default Stats Collector is the :class:`MemoryStatsCollector` is used.
|
||||
|
||||
When stats are disabled (through the :setting:`STATS_ENABLED` setting) the
|
||||
:setting:`STATS_CLASS` setting is ignored and the :class:`DummyStatsCollector`
|
||||
is used.
|
||||
|
||||
MemoryStatsCollector
|
||||
--------------------
|
||||
|
||||
.. class:: MemoryStatsCollector
|
||||
|
||||
A simple stats collector that keeps the stats of the last scraping run (for
|
||||
each domain) in memory, which can be accessed through the ``domain_stats``
|
||||
attribute
|
||||
|
||||
This is the default Stats Collector used in Scrapy.
|
||||
|
||||
.. attribute:: domain_stats
|
||||
|
||||
A dict of dicts (keyed by domain) containing the stats of the last
|
||||
scraping run for each domain.
|
||||
|
||||
DummyStatsCollector
|
||||
-------------------
|
||||
|
||||
.. class:: DummyStatsCollector
|
||||
|
||||
A Stats collector which does nothing but is very efficient. This is the
|
||||
Stats Collector used when stats are diabled (through the
|
||||
:setting:`STATS_ENABLED` setting).
|
||||
|
||||
SimpledbStatsCollector
|
||||
----------------------
|
||||
|
||||
.. module:: scrapy.stats.collector.simpledb
|
||||
:synopsis: Simpledb Stats Collector
|
||||
|
||||
.. class:: SimpledbStatsCollector
|
||||
|
||||
A Stats collector which persists stats to `Amazon SimpleDB`_, using one
|
||||
SimpleDB item per scraping run (ie. it keeps history of all scraping runs).
|
||||
The data is persisted to the SimpleDB domain specified by the
|
||||
:setting:`STATS_SDB_DOMAIN` setting.
|
||||
|
||||
In addition to the existing stats keys the following keys are added at
|
||||
persitance time:
|
||||
|
||||
* ``domain``: the spider domain (so you can use it later for querying stats
|
||||
for that domain)
|
||||
* ``timestamp``: the timestamp when the stats were persisited
|
||||
|
||||
Both the ``domain`` and ``timestamp`` are used for generating the SimpleDB
|
||||
item name in order to avoid overwriting stats of previous scraping runs.
|
||||
|
||||
As `required by SimpleDB`_, datetime's are stored in ISO 8601 format and
|
||||
numbers are zero-padded to 16 digits. Negative numbers are not currently
|
||||
supported.
|
||||
|
||||
This Stats Collector requires the `boto`_ library.
|
||||
|
||||
.. _Amazon SimpleDB: http://aws.amazon.com/simpledb/
|
||||
.. _required by SimpleDB: http://docs.amazonwebservices.com/AmazonSimpleDB/2009-04-15/DeveloperGuide/ZeroPadding.html
|
||||
.. _boto: http://code.google.com/p/boto/
|
||||
|
||||
This Stats Collector can be configured through the following settings:
|
||||
|
||||
.. setting:: STATS_SDB_DOMAIN
|
||||
|
||||
STATS_SDB_DOMAIN
|
||||
~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``'scrapy_stats'``
|
||||
|
||||
A string containing the SimpleDB domain to use in the
|
||||
:class:`SimpledbStatsCollector`.
|
||||
|
||||
.. setting:: STATS_SDB_ASYNC
|
||||
|
||||
STATS_SDB_ASYNC
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``False``
|
||||
|
||||
If ``True`` communication with SimpleDB will be performed asynchronously. If
|
||||
``False`` blocking IO will be used instead. This is the default as using
|
||||
asynchronous communication can result in the stats not being persisted if the
|
||||
Scrapy engine is shut down in the middle (for example, when you run only one
|
||||
spider in a process and then exit).
|
||||
|
||||
Stats signals
|
||||
=============
|
||||
|
||||
The Stats Collector provides some signals for extending the stats collection
|
||||
functionality:
|
||||
|
||||
.. module:: scrapy.stats.signals
|
||||
:synopsis: Stats Collector signals
|
||||
|
||||
.. signal:: stats_domain_opened
|
||||
.. function:: stats_domain_opened(domain)
|
||||
|
||||
Sent right after the stats domain is opened. You can use this signal to add
|
||||
startup stats for domain (example: start time).
|
||||
|
||||
:param domain: the stats domain just opened
|
||||
:type domain: str
|
||||
|
||||
.. signal:: stats_domain_closing
|
||||
.. function:: stats_domain_closing(domain, reason)
|
||||
|
||||
Sent just before the stats domain is closed. You can use this signal to add
|
||||
some closing stats (example: finish time).
|
||||
|
||||
:param domain: the stats domain about to be closed
|
||||
:type domain: str
|
||||
|
||||
:param reason: the reason why the domain is being closed. See
|
||||
:signal:`domain_closed` signal for more info.
|
||||
:type reason: str
|
||||
|
||||
.. signal:: stats_domain_closed
|
||||
.. function:: stats_domain_closed(domain, reason)
|
||||
|
||||
Sent right after the stats domain is closed. You can use this signal to
|
||||
collect resources.
|
||||
|
||||
:param domain: the stats domain just closed
|
||||
:type domain: str
|
||||
|
||||
:param reason: the reason why the domain is being closed. See
|
||||
:signal:`domain_closed` signal for more info.
|
||||
:type reason: str
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
import pprint
|
||||
from scrapy.command import ScrapyCommand
|
||||
from scrapy.conf import settings
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
def syntax(self):
|
||||
return "<domain> [domain ...]"
|
||||
|
||||
def short_desc(self):
|
||||
return "Show all stats history stored for the given domain(s)"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("-p", "--path", dest="path", help="restrict stats to PATH", metavar="PATH")
|
||||
parser.add_option("--limit", dest="limit", default='10', help="limit the number of stats")
|
||||
|
||||
def run(self, args, opts):
|
||||
if not args:
|
||||
print "A domain is required"
|
||||
return
|
||||
|
||||
if not settings['SCRAPING_DB']:
|
||||
print "SCRAPING_DB setting is required for this command"
|
||||
return
|
||||
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history')
|
||||
|
||||
for domain in args:
|
||||
print "# %s" % domain
|
||||
pprint.pprint(list(ddh.get(domain, path=opts.path, count=int(opts.limit))))
|
||||
|
||||
|
|
@ -184,10 +184,13 @@ SPIDER_MIDDLEWARES_BASE = {
|
|||
# Spider side
|
||||
}
|
||||
|
||||
STATS_CLASS = 'scrapy.stats.collector.MemoryStatsCollector'
|
||||
STATS_ENABLED = True
|
||||
STATS_CLEANUP = False
|
||||
STATS_DEBUG = False
|
||||
|
||||
STATS_SDB_DOMAIN = 'scrapy_stats'
|
||||
STATS_SDB_ASYNC = False
|
||||
|
||||
# deprecated settings - the stats web service should be moved to the web console
|
||||
STATS_WSPORT = 8089
|
||||
STATS_WSTIMEOUT = 15
|
||||
|
|
|
|||
|
|
@ -6,12 +6,6 @@ class DownloaderStats(object):
|
|||
"""DownloaderStats store stats of all requests, responses and
|
||||
exceptions that pass through it.
|
||||
|
||||
They are stored in the following keys:
|
||||
|
||||
SPIDER/downloader/request_method_count
|
||||
SPIDER/downloader/response_status_count
|
||||
SPIDER/downloader/exception_count
|
||||
|
||||
To use this middleware you must enable the DOWNLOADER_STATS setting.
|
||||
"""
|
||||
|
||||
|
|
@ -20,27 +14,26 @@ class DownloaderStats(object):
|
|||
raise NotConfigured
|
||||
|
||||
def process_request(self, request, spider):
|
||||
stats.incpath('_global/downloader/request_count')
|
||||
stats.incpath('%s/downloader/request_count' % spider.domain_name)
|
||||
stats.incpath('%s/downloader/request_method_count/%s' % (spider.domain_name, request.method))
|
||||
domain = spider.domain_name
|
||||
stats.inc_value('downloader/request_count')
|
||||
stats.inc_value('downloader/request_count', domain=domain)
|
||||
stats.inc_value('downloader/request_method_count/%s' % request.method, domain=domain)
|
||||
reqlen = len(request.httprepr())
|
||||
stats.incpath('%s/downloader/request_bytes' % spider.domain_name, reqlen)
|
||||
stats.incpath('_global/downloader/request_bytes', reqlen)
|
||||
stats.inc_value('downloader/request_bytes', reqlen, domain=domain)
|
||||
stats.inc_value('downloader/request_bytes', reqlen)
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
self._inc_response_count(response, spider.domain_name)
|
||||
domain = spider.domain_name
|
||||
stats.inc_value('downloader/response_count')
|
||||
stats.inc_value('downloader/response_count', domain=domain)
|
||||
stats.inc_value('downloader/response_status_count/%s' % response.status, domain=domain)
|
||||
reslen = len(response.httprepr())
|
||||
stats.inc_value('downloader/response_bytes', reslen, domain=domain)
|
||||
stats.inc_value('downloader/response_bytes', reslen)
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)
|
||||
stats.incpath('_global/downloader/exception_count')
|
||||
stats.incpath('%s/downloader/exception_count' % spider.domain_name)
|
||||
stats.incpath('%s/downloader/exception_type_count/%s' % (spider.domain_name, ex_class))
|
||||
|
||||
def _inc_response_count(self, response, domain):
|
||||
stats.incpath('_global/downloader/response_count')
|
||||
stats.incpath('%s/downloader/response_count' % domain)
|
||||
stats.incpath('%s/downloader/response_status_count/%s' % (domain, response.status))
|
||||
reslen = len(response.httprepr())
|
||||
stats.incpath('%s/downloader/response_bytes' % domain, reslen)
|
||||
stats.incpath('_global/downloader/response_bytes', reslen)
|
||||
stats.inc_value('downloader/exception_count')
|
||||
stats.inc_value('downloader/exception_count', domain=spider.domain_name)
|
||||
stats.inc_value('downloader/exception_type_count/%s' % ex_class, domain=spider.domain_name)
|
||||
|
|
|
|||
|
|
@ -52,11 +52,11 @@ class ItemSamplerPipeline(object):
|
|||
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
|
||||
|
||||
def process_item(self, domain, item):
|
||||
sampled = stats.getpath("%s/items_sampled" % domain, 0)
|
||||
sampled = stats.get_value("items_sampled", 0, domain=domain)
|
||||
if sampled < items_per_domain:
|
||||
self.items[item.guid] = item
|
||||
sampled += 1
|
||||
stats.setpath("%s/items_sampled" % domain, sampled)
|
||||
stats.set_value("items_sampled", sampled, domain=domain)
|
||||
log.msg("Sampled %s" % item, domain=domain, level=log.INFO)
|
||||
if close_domain and sampled == items_per_domain:
|
||||
scrapyengine.close_domain(domain)
|
||||
|
|
@ -69,7 +69,7 @@ class ItemSamplerPipeline(object):
|
|||
log.msg("No products sampled for: %s" % " ".join(self.empty_domains), level=log.WARNING)
|
||||
|
||||
def domain_closed(self, domain, spider, reason):
|
||||
if reason == 'finished' and not stats.getpath("%s/items_sampled" % domain):
|
||||
if reason == 'finished' and not stats.get_value("items_sampled", domain=domain):
|
||||
self.empty_domains.add(domain)
|
||||
self.domains_count += 1
|
||||
log.msg("Sampled %d domains so far (%d empty)" % (self.domains_count, len(self.empty_domains)), level=log.INFO)
|
||||
|
|
@ -84,7 +84,7 @@ class ItemSamplerMiddleware(object):
|
|||
raise NotConfigured
|
||||
|
||||
def process_spider_input(self, response, spider):
|
||||
if stats.getpath("%s/items_sampled" % spider.domain_name) >= items_per_domain:
|
||||
if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_domain:
|
||||
return []
|
||||
elif max_response_size and max_response_size > len(response.httprepr()):
|
||||
return []
|
||||
|
|
@ -97,7 +97,7 @@ class ItemSamplerMiddleware(object):
|
|||
else:
|
||||
items.append(r)
|
||||
|
||||
if stats.getpath("%s/items_sampled" % spider.domain_name) >= items_per_domain:
|
||||
if stats.get_value("items_sampled", domain=spider.domain_name) >= items_per_domain:
|
||||
return []
|
||||
else:
|
||||
# TODO: this needs some revision, as keeping only the first item
|
||||
|
|
|
|||
|
|
@ -156,8 +156,8 @@ class BaseImagesPipeline(MediaPipeline):
|
|||
yield thumb_key, thumb_image, thumb_buf
|
||||
|
||||
def inc_stats(self, domain, status):
|
||||
stats.incpath('%s/image_count' % domain)
|
||||
stats.incpath('%s/image_status_count/%s' % (domain, status))
|
||||
stats.inc_value('image_count', domain=domain)
|
||||
stats.inc_value('image_status_count/%s' % status, domain=domain)
|
||||
|
||||
def convert_image(self, image, size=None):
|
||||
if image.mode != 'RGB':
|
||||
|
|
|
|||
|
|
@ -43,16 +43,17 @@ class SpiderProfiler(object):
|
|||
tafter = datetime.datetime.now()
|
||||
mafter = self._memusage()
|
||||
ct = tafter-tbefore
|
||||
tcc = stats.getpath('%s/profiling/total_callback_time' % spider.domain_name, datetime.timedelta(0))
|
||||
sct = stats.getpath('%s/profiling/slowest_callback_time' % spider.domain_name, datetime.timedelta(0))
|
||||
stats.setpath('%s/profiling/total_callback_time' % spider.domain_name, tcc+ct)
|
||||
domain = spider.domain_name
|
||||
tcc = stats.get_value('profiling/total_callback_time', datetime.timedelta(0), domain=domain)
|
||||
sct = stats.get_value('profiling/slowest_callback_time', datetime.timedelta(0), domain=domain)
|
||||
stats.set_value('profiling/total_callback_time' % spider.domain_name, tcc+ct, domain=domain)
|
||||
if ct > sct:
|
||||
stats.setpath('%s/profiling/slowest_callback_time' % spider.domain_name, ct)
|
||||
stats.setpath('%s/profiling/slowest_callback_name' % spider.domain_name, function.__name__)
|
||||
stats.setpath('%s/profiling/slowest_callback_url' % spider.domain_name, args[0].url)
|
||||
stats.set_value('profiling/slowest_callback_time', ct, domain=domain)
|
||||
stats.set_value('profiling/slowest_callback_name', function.__name__, domain=domain)
|
||||
stats.set_value('profiling/slowest_callback_url', args[0].url, domain=domain)
|
||||
if self.memusage:
|
||||
tma = stats.getpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, 0)
|
||||
stats.setpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, tma+mafter-mbefore)
|
||||
tma = stats.get_value('profiling/total_mem_allocated_in_callbacks', 0, domain=domain)
|
||||
stats.set_value('profiling/total_mem_allocated_in_callbacks', tma+mafter-mbefore, domain=domain)
|
||||
return r
|
||||
return new_callback
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class DepthMiddleware(object):
|
|||
self.maxdepth = settings.getint('DEPTH_LIMIT')
|
||||
self.stats = settings.getbool('DEPTH_STATS')
|
||||
if self.stats and self.maxdepth:
|
||||
stats.setpath('_envinfo/request_depth_limit', self.maxdepth)
|
||||
stats.set_value('envinfo/request_depth_limit', self.maxdepth)
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
def _filter(request):
|
||||
|
|
@ -26,13 +26,13 @@ class DepthMiddleware(object):
|
|||
log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name)
|
||||
return False
|
||||
elif self.stats:
|
||||
stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, depth))
|
||||
if depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0):
|
||||
stats.setpath('%s/request_depth_max' % spider.domain_name, depth)
|
||||
stats.inc_value('request_depth_count/%s' % depth, domain=spider.domain_name)
|
||||
if depth > stats.get_value('request_depth_max', 0, domain=spider.domain_name):
|
||||
stats.set_value('request_depth_max', depth, domain=spider.domain_name)
|
||||
return True
|
||||
|
||||
if self.stats and 'depth' not in response.request.meta: # otherwise we loose stats for depth=0
|
||||
response.request.meta['depth'] = 0
|
||||
stats.incpath('%s/request_depth_count/0' % spider.domain_name)
|
||||
stats.inc_value('request_depth_count/0', domain=spider.domain_name)
|
||||
|
||||
return (r for r in result or () if _filter(r))
|
||||
|
|
|
|||
|
|
@ -1,55 +0,0 @@
|
|||
from functools import partial
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.utils.serialization import serialize
|
||||
from scrapy.conf import settings
|
||||
|
||||
from .site import WebSite, WebResource
|
||||
from .json import JsonResponse
|
||||
|
||||
STATS_WSPORT = settings.getint('STATS_WSPORT', 8089)
|
||||
STATS_WSTIMEOUT = settings.getint('STATS_WSTIMEOUT', 15)
|
||||
|
||||
jsonserialize = partial(serialize, format='json')
|
||||
|
||||
|
||||
def stats(request):
|
||||
service = request.ARGS.get('service', 'getstats')
|
||||
if service == 'getstats':
|
||||
domain = request.ARGS.get('domain')
|
||||
count = request.ARGS.get('count', 1)
|
||||
if not domain:
|
||||
path = request.ARGS.get('path')
|
||||
offset = request.ARGS.get('offset')
|
||||
order = request.ARGS.get('order', 'domain')
|
||||
olist = request.ARGS.get('olist', 'ASC')
|
||||
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history')
|
||||
|
||||
if service == 'countdomains':
|
||||
stats = [ddh.domain_count()]
|
||||
elif domain:
|
||||
stats = list(ddh.get(domain, count=int(count)))
|
||||
else:
|
||||
stats = list(ddh.getlast_alldomains(count, offset, order, olist, path=path))
|
||||
|
||||
content = {'data': stats if stats else "No stats found"}
|
||||
return JsonResponse(content=content, serialize=jsonserialize)
|
||||
|
||||
|
||||
urlmapping = (
|
||||
('^ws/tools/stats/$', stats),
|
||||
)
|
||||
|
||||
|
||||
class StatsService(WebSite):
|
||||
def __init__(self):
|
||||
if not settings['WS_ENABLED']:
|
||||
raise NotConfigured
|
||||
|
||||
if not settings['SCRAPING_DB']:
|
||||
print "SCRAPING_DB setting is required for the stats web service"
|
||||
raise NotConfigured
|
||||
|
||||
resource = WebResource(urlmapping, timeout=STATS_WSTIMEOUT)
|
||||
WebSite.__init__(self, port=STATS_WSPORT, resource=resource)
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
"""
|
||||
Extension for persistent statistics by domain
|
||||
"""
|
||||
import re
|
||||
import pprint
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.core.exceptions import NotConfigured
|
||||
from scrapy.management.web import banner
|
||||
from scrapy.conf import settings
|
||||
|
||||
class SpiderStats(object):
|
||||
webconsole_id = 'spiderstats'
|
||||
webconsole_name = 'Spider stats (all-time)'
|
||||
|
||||
stats_mainpage = [
|
||||
('item_scraped_count', "Items scraped <br>(last run)"),
|
||||
('item_passed_count', "Items passed <br>(last run)"),
|
||||
('response_count', "Pages crawled <br>(last run)"),
|
||||
('start_time', "Start time <br>(last run)"),
|
||||
('finish_time', "Finish time <br>(last run)"),
|
||||
('finish_status', "Finish status<br>(last run)"),
|
||||
]
|
||||
|
||||
PATH_RE = re.compile("/spiderstats/([^/]+)")
|
||||
|
||||
def __init__(self):
|
||||
if not settings['SCRAPING_DB']:
|
||||
raise NotConfigured("SpiderStats: Requires SCRAPING_DB setting")
|
||||
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
self.ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history')
|
||||
|
||||
from scrapy.management.web import webconsole_discover_module
|
||||
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
|
||||
|
||||
def webconsole_render(self, wc_request):
|
||||
m = self.PATH_RE.search(wc_request.path)
|
||||
if m:
|
||||
return self.render_domain(m.group(1))
|
||||
else:
|
||||
return self.render_main()
|
||||
|
||||
def render_main(self):
|
||||
s = banner(self)
|
||||
s += "<table border='1'>\n"
|
||||
s += "<tr>\n"
|
||||
s += "<th>Domain</th>"
|
||||
for path, title in self.stats_mainpage:
|
||||
s += "<th>%s</th>" % title
|
||||
s += "</tr>\n"
|
||||
for domain in spiders.asdict().keys():
|
||||
s += "<tr>\n"
|
||||
s += "<td><a href='%s'>%s</a></td>" % (domain, domain)
|
||||
for path, title in self.stats_mainpage:
|
||||
stat = self.ddh.getlast(domain, path=path)
|
||||
value = stat[1] if stat else "None"
|
||||
s += "<td>%s</td>" % value
|
||||
s += "</tr>\n"
|
||||
s += "</table>\n"
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return str(s)
|
||||
|
||||
def render_domain(self, domain):
|
||||
s = banner(self)
|
||||
stats = self.ddh.getall(domain)
|
||||
s += "<pre>\n"
|
||||
s += pprint.pformat(list(stats))
|
||||
s += "</pre>\n"
|
||||
s += "</body>\n"
|
||||
s += "</html>\n"
|
||||
|
||||
return str(s)
|
||||
|
||||
def webconsole_discover_module(self):
|
||||
return self
|
||||
|
|
@ -129,8 +129,8 @@ class Scraper(object):
|
|||
referer = request.headers.get('Referer', None)
|
||||
msg = "SPIDER BUG processing <%s> from <%s>: %s" % (request.url, referer, _failure)
|
||||
log.msg(msg, log.ERROR, domain=spider.domain_name)
|
||||
stats.incpath("%s/spider_exceptions/%s" % (spider.domain_name, \
|
||||
_failure.value.__class__.__name__))
|
||||
stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \
|
||||
domain=spider.domain_name)
|
||||
|
||||
def handle_spider_output(self, result, request, response, spider):
|
||||
if not result:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,9 @@
|
|||
from scrapy.stats.statscollector import StatsCollector
|
||||
from scrapy.stats.collector import DummyStatsCollector
|
||||
from scrapy.conf import settings
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
stats = StatsCollector()
|
||||
# if stats are disabled use a DummyStatsCollector to improve performance
|
||||
if settings.getbool('STATS_ENABLED'):
|
||||
stats = load_object(settings['STATS_CLASS'])()
|
||||
else:
|
||||
stats = DummyStatsCollector()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
"""
|
||||
Scrapy extension for collecting scraping stats
|
||||
"""
|
||||
import pprint
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy.stats.signals import stats_domain_opened, stats_domain_closing, \
|
||||
stats_domain_closed
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.conf import settings
|
||||
|
||||
class StatsCollector(object):
|
||||
|
||||
def __init__(self):
|
||||
self.debug = settings.getbool('STATS_DEBUG')
|
||||
self._stats = {None: {}} # None is for global stats
|
||||
|
||||
dispatcher.connect(self.open_domain, signal=signals.domain_open)
|
||||
dispatcher.connect(self.close_domain, signal=signals.domain_closed)
|
||||
|
||||
def get_value(self, key, default=None, domain=None):
|
||||
return self._stats[domain].get(key, default)
|
||||
|
||||
def get_stats(self, domain=None):
|
||||
return self._stats[domain]
|
||||
|
||||
def set_value(self, key, value, domain=None):
|
||||
self._stats[domain][key] = value
|
||||
|
||||
def set_stats(self, stats, domain=None):
|
||||
self._stats[domain] = stats
|
||||
|
||||
def inc_value(self, key, count=1, start=0, domain=None):
|
||||
d = self._stats[domain]
|
||||
d[key] = d.setdefault(key, start) + count
|
||||
|
||||
def clear_stats(self, domain=None):
|
||||
self._stats[domain].clear()
|
||||
|
||||
def list_domains(self):
|
||||
return [d for d in self._stats.keys() if d is not None]
|
||||
|
||||
def open_domain(self, domain):
|
||||
self._stats[domain] = {}
|
||||
signals.send_catch_log(stats_domain_opened, domain=domain)
|
||||
|
||||
def close_domain(self, domain, reason):
|
||||
signals.send_catch_log(stats_domain_closing, domain=domain, reason=reason)
|
||||
if self.debug:
|
||||
log.msg(pprint.pformat(self[domain]), domain=domain, level=log.DEBUG)
|
||||
del self._stats[domain]
|
||||
signals.send_catch_log(stats_domain_closed, domain=domain, reason=reason)
|
||||
|
||||
|
||||
class MemoryStatsCollector(StatsCollector):
|
||||
|
||||
def __init__(self):
|
||||
super(MemoryStatsCollector, self).__init__()
|
||||
self.domain_stats = {}
|
||||
|
||||
def close_domain(self, domain, reason):
|
||||
self.domain_stats[domain] = self._stats[domain]
|
||||
super(MemoryStatsCollector, self).close_domain(domain, reason)
|
||||
|
||||
|
||||
class DummyStatsCollector(StatsCollector):
|
||||
|
||||
def get_value(self, key, default=None, domain=None):
|
||||
return default
|
||||
|
||||
def set_value(self, key, value, domain=None):
|
||||
pass
|
||||
|
||||
def set_stats(self, stats, domain=None):
|
||||
pass
|
||||
|
||||
def inc_value(self, key, count=1, domain=None):
|
||||
pass
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
"""
|
||||
A Stats collector for persisting stats (pickled) to a MySQL db
|
||||
"""
|
||||
|
||||
import cPickle as pickle
|
||||
from datetime import datetime
|
||||
|
||||
from scrapy.stats.collector import StatsCollector
|
||||
from scrapy.utils.db import mysql_connect
|
||||
from scrapy.conf import settings
|
||||
|
||||
class MysqlStatsCollector(StatsCollector):
|
||||
|
||||
def __init__(self):
|
||||
super(MysqlStatsCollector, self).__init__()
|
||||
mysqluri = settings['STATS_MYSQL_URI']
|
||||
self._mysql_conn = mysql_connect(mysqluri, use_unicode=False) if mysqluri else None
|
||||
|
||||
def close_domain(self, domain, reason):
|
||||
if self._mysql_conn:
|
||||
stored = datetime.now()
|
||||
datas = pickle.dumps(self._stats[domain])
|
||||
table = 'domain_data_history'
|
||||
|
||||
c = self._mysql_conn.cursor()
|
||||
c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % table, \
|
||||
(domain, stored, datas))
|
||||
self._mysql_conn.commit()
|
||||
super(MysqlStatsCollector, self).close_domain(domain, reason)
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
A Stats collector for persisting stats to Amazon SimpleDB.
|
||||
|
||||
Requires the boto library: http://code.google.com/p/boto/
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import boto
|
||||
|
||||
from scrapy.stats.collector import StatsCollector
|
||||
from scrapy import log
|
||||
from scrapy.conf import settings
|
||||
|
||||
class SimpledbStatsCollector(StatsCollector):
|
||||
"""A simple in-memory stats collector which keeps scraping stats of last
|
||||
run. Those can be accessed through the ``domain_stats`` attribute"""
|
||||
|
||||
def __init__(self):
|
||||
super(SimpledbStatsCollector, self).__init__()
|
||||
self._sdbdomain = settings['STATS_SDB_DOMAIN']
|
||||
self._async = settings.getbool('STATS_SDB_ASYNC')
|
||||
|
||||
def close_domain(self, domain, reason):
|
||||
if self._sdbdomain:
|
||||
if self._async:
|
||||
from twisted.internet import threads
|
||||
dfd = threads.deferToThread(self._persist_to_sdb, domain, \
|
||||
self._stats[domain].copy())
|
||||
dfd.addErrback(log.err, 'Error uploading stats to SimpleDB', domain=domain)
|
||||
else:
|
||||
self._persist_to_sdb(domain, self._stats[domain])
|
||||
super(SimpledbStatsCollector, self).close_domain(domain, reason)
|
||||
|
||||
def _persist_to_sdb(self, domain, stats):
|
||||
ts = datetime.now().isoformat()
|
||||
sdb_item_id = "%s_%s" % (domain, ts)
|
||||
sdb_item = dict([(k, self._to_sdb_value(v)) for k, v in stats.iteritems()])
|
||||
sdb_item['domain'] = domain
|
||||
sdb_item['timestamp'] = self._to_sdb_value(ts)
|
||||
sdb = boto.connect_sdb()
|
||||
sdb.create_domain(self._sdbdomain)
|
||||
sdb.batch_put_attributes(self._sdbdomain, {sdb_item_id: sdb_item})
|
||||
|
||||
def _to_sdb_value(self, obj):
|
||||
if isinstance(obj, int):
|
||||
return "%016d" % obj
|
||||
elif isinstance(obj, datetime):
|
||||
return obj.isoformat()
|
||||
elif isinstance(obj, basestring):
|
||||
return obj
|
||||
else:
|
||||
raise TypeError("SimpledbStatsCollector unsupported type: %s" % \
|
||||
type(obj).__name__)
|
||||
|
|
@ -10,43 +10,44 @@ from scrapy.xlib.pydispatch import dispatcher
|
|||
|
||||
from scrapy.core import signals
|
||||
from scrapy.stats import stats
|
||||
from scrapy.stats.signals import stats_domain_opened, stats_domain_closing
|
||||
from scrapy.conf import settings
|
||||
|
||||
class CoreStats(object):
|
||||
"""Scrapy core stats collector"""
|
||||
|
||||
def __init__(self):
|
||||
stats.setpath('_envinfo/user', getpass.getuser())
|
||||
stats.setpath('_envinfo/host', socket.gethostname())
|
||||
stats.setpath('_envinfo/logfile', settings['LOGFILE'])
|
||||
stats.setpath('_envinfo/pid', os.getpid())
|
||||
stats.set_value('envinfo/user', getpass.getuser())
|
||||
stats.set_value('envinfo/host', socket.gethostname())
|
||||
stats.set_value('envinfo/logfile', settings['LOGFILE'])
|
||||
stats.set_value('envinfo/pid', os.getpid())
|
||||
|
||||
dispatcher.connect(self.stats_domain_open, signal=stats.domain_open)
|
||||
dispatcher.connect(self.stats_domain_closing, signal=stats.domain_closing)
|
||||
dispatcher.connect(self.stats_domain_opened, signal=stats_domain_opened)
|
||||
dispatcher.connect(self.stats_domain_closing, signal=stats_domain_closing)
|
||||
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
|
||||
dispatcher.connect(self.item_passed, signal=signals.item_passed)
|
||||
dispatcher.connect(self.item_dropped, signal=signals.item_dropped)
|
||||
|
||||
def stats_domain_open(self, domain, spider):
|
||||
stats.setpath('%s/start_time' % domain, datetime.datetime.now())
|
||||
stats.setpath('%s/envinfo' % domain, stats.getpath('_envinfo'))
|
||||
stats.incpath('_global/domain_count/opened')
|
||||
def stats_domain_opened(self, domain):
|
||||
stats.set_value('start_time', datetime.datetime.now(), domain=domain)
|
||||
stats.set_value('envinfo/host', stats.get_value('envinfo/host'), domain=domain)
|
||||
stats.inc_value('domain_count/opened')
|
||||
|
||||
def stats_domain_closing(self, domain, spider, reason):
|
||||
stats.setpath('%s/finish_time' % domain, datetime.datetime.now())
|
||||
stats.setpath('%s/finish_status' % domain, 'OK' if reason == 'finished' else reason)
|
||||
stats.incpath('_global/domain_count/%s' % reason)
|
||||
def stats_domain_closing(self, domain, reason):
|
||||
stats.set_value('finish_time', datetime.datetime.now(), domain=domain)
|
||||
stats.set_value('finish_status', 'OK' if reason == 'finished' else reason, domain=domain)
|
||||
stats.inc_value('domain_count/%s' % reason, domain=domain)
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
stats.incpath('%s/item_scraped_count' % spider.domain_name)
|
||||
stats.incpath('_global/item_scraped_count')
|
||||
stats.inc_value('item_scraped_count', domain=spider.domain_name)
|
||||
stats.inc_value('item_scraped_count')
|
||||
|
||||
def item_passed(self, item, spider):
|
||||
stats.incpath('%s/item_passed_count' % spider.domain_name)
|
||||
stats.incpath('_global/item_passed_count')
|
||||
stats.inc_value('item_passed_count', domain=spider.domain_name)
|
||||
stats.inc_value('item_passed_count')
|
||||
|
||||
def item_dropped(self, item, spider, exception):
|
||||
reason = exception.__class__.__name__
|
||||
stats.incpath('%s/item_dropped_count' % spider.domain_name)
|
||||
stats.incpath('%s/item_dropped_reasons_count/%s' % (spider.domain_name, reason))
|
||||
stats.incpath('_global/item_dropped_count')
|
||||
stats.inc_value('item_dropped_count', domain=spider.domain_name)
|
||||
stats.inc_value('item_dropped_reasons_count/%s' % reason, domain=spider.domain_name)
|
||||
stats.inc_value('item_dropped_count')
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
stats_domain_opened = object()
|
||||
stats_domain_closing = object()
|
||||
stats_domain_closed = object()
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
"""
|
||||
Scrapy extension for collecting scraping stats
|
||||
"""
|
||||
import pprint
|
||||
|
||||
from scrapy.xlib.pydispatch import dispatcher
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy import log
|
||||
from scrapy.utils.misc import stats_getpath
|
||||
from scrapy.conf import settings
|
||||
|
||||
class StatsCollector(dict):
|
||||
# signal sent after a domain is opened for stats collection and its resorces have been allocated. args: domain
|
||||
domain_open = object()
|
||||
# signal sent before a domain is closed, and before the stats are persisted. it can be catched to add additional stats. args: domain
|
||||
domain_closing = object()
|
||||
# signal sent after a domain is closed and its resources have been freed. args: domain
|
||||
domain_closed = object()
|
||||
|
||||
def __init__(self, enabled=None):
|
||||
self.db = None
|
||||
self.debug = settings.getbool('STATS_DEBUG')
|
||||
self.enabled = enabled if enabled is not None else settings.getbool('STATS_ENABLED')
|
||||
self.cleanup = settings.getbool('STATS_CLEANUP')
|
||||
|
||||
if self.enabled:
|
||||
if settings['SCRAPING_DB']:
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
self.db = DomainDataHistory(settings['SCRAPING_DB'], table_name='domain_data_history')
|
||||
|
||||
dispatcher.connect(self._domain_open, signal=signals.domain_open)
|
||||
dispatcher.connect(self._domain_closed, signal=signals.domain_closed)
|
||||
else:
|
||||
self.setpath = lambda *args, **kwargs: None
|
||||
self.getpath = lambda path, default=None: default
|
||||
self.incpath = lambda *args, **kwargs: None
|
||||
self.delpath = lambda *args, **kwargs: None
|
||||
|
||||
def setpath(self, path, value):
|
||||
d = self
|
||||
keys = path.split('/')
|
||||
for key in keys[:-1]:
|
||||
if key not in d:
|
||||
d[key] = {}
|
||||
d = d[key]
|
||||
d[keys[-1]] = value
|
||||
|
||||
def getpath(self, path, default=None):
|
||||
return stats_getpath(self, path, default)
|
||||
|
||||
def delpath(self, path):
|
||||
d = self
|
||||
keys = path.split('/')
|
||||
for key in keys[:-1]:
|
||||
if key in d:
|
||||
d = d[key]
|
||||
else:
|
||||
return
|
||||
del d[keys[-1]]
|
||||
|
||||
def incpath(self, path, value_diff=1):
|
||||
curvalue = self.getpath(path) or 0
|
||||
self.setpath(path, curvalue + value_diff)
|
||||
|
||||
def _domain_open(self, domain, spider):
|
||||
dispatcher.send(signal=self.domain_open, sender=self.__class__, domain=domain, spider=spider)
|
||||
|
||||
def _domain_closed(self, domain, spider, reason):
|
||||
dispatcher.send(signal=self.domain_closing, sender=self.__class__, domain=domain, spider=spider, reason=reason)
|
||||
if self.debug:
|
||||
log.msg(pprint.pformat(self[domain]), domain=domain, level=log.DEBUG)
|
||||
if self.db:
|
||||
self.db.put(domain, self[domain])
|
||||
if self.cleanup:
|
||||
del self[domain]
|
||||
dispatcher.send(signal=self.domain_closed, sender=self.__class__, domain=domain, spider=spider, reason=reason)
|
||||
|
|
@ -1,130 +0,0 @@
|
|||
"""
|
||||
Store implementations using a MySQL DB
|
||||
"""
|
||||
import cPickle as pickle
|
||||
from datetime import datetime
|
||||
|
||||
from scrapy.utils.misc import stats_getpath
|
||||
from scrapy.utils.db import mysql_connect
|
||||
|
||||
class DomainDataHistory(object):
|
||||
"""
|
||||
This is a store for domain data, with history.
|
||||
"""
|
||||
|
||||
def __init__(self, db_uri, table_name):
|
||||
self.db_uri = db_uri
|
||||
self._mysql_conn = None
|
||||
self._table = table_name
|
||||
|
||||
def get_mysql_conn(self):
|
||||
if self._mysql_conn is None:
|
||||
self._mysql_conn = mysql_connect(self.db_uri, use_unicode=False)
|
||||
return self._mysql_conn
|
||||
mysql_conn = property(get_mysql_conn)
|
||||
|
||||
def get(self, domain, count=1, offset=0, stored_after=None, stored_before=None, path=None):
|
||||
"""
|
||||
Get the last records stored for the given domain.
|
||||
If count is given that number of records is retunred, otherwise all records.
|
||||
If stored_{after,before} is given the results are restricted to that time interval.
|
||||
The result is a list of tuples: (timestamp, data)
|
||||
If path is given it only returns stats of that given path (only for dict objects like stats)
|
||||
"""
|
||||
sqlsuf = ""
|
||||
if count is not None:
|
||||
sqlsuf += "LIMIT %d " % count
|
||||
if offset:
|
||||
sqlsuf += "OFFSET %d " % offset
|
||||
filter = ""
|
||||
if stored_after:
|
||||
filter += "AND stored>='%s'" % stored_after.strftime("%Y-%m-%d %H:%M:%S")
|
||||
if stored_before:
|
||||
filter += "AND stored<='%s'" % stored_before.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
c = self.mysql_conn.cursor()
|
||||
select = "SELECT stored,data FROM %s WHERE domain=%%s %s ORDER BY stored DESC %s" % (self._table, filter, sqlsuf)
|
||||
c.execute(select, domain)
|
||||
for stored, datas in c:
|
||||
obj = pickle.loads(datas)
|
||||
if path and isinstance(obj, dict):
|
||||
yield stored, stats_getpath(obj, path)
|
||||
else:
|
||||
yield stored, obj
|
||||
|
||||
def getlast(self, domain, offset=0, path=None):
|
||||
"""
|
||||
Get the Nth last data stored for the given domain
|
||||
If offset=0 get the last data stored
|
||||
If offset=1 get the previous data stored
|
||||
...and so on
|
||||
"""
|
||||
datal = list(self.get(domain, count=1, offset=offset, path=path))
|
||||
if datal:
|
||||
return datal[0]
|
||||
|
||||
def getall(self, domain, path=None):
|
||||
"""
|
||||
Get all data stored for the given domain as a list of tuples:
|
||||
(stored, timestamp)
|
||||
"""
|
||||
return self.get(domain, count=None, path=path)
|
||||
|
||||
def getlast_alldomains(self, count=None, offset=None, order='domain', olist='ASC', path=None):
|
||||
"""
|
||||
Get all last data for all domains as a list of tuples, unless
|
||||
data is sliced by count and offset values.
|
||||
Ordered by domain is the default.
|
||||
TODO: This could be merged with: get(domain=None, ..., +order)
|
||||
|
||||
Return a list of tuples (domain, timestamp, data)
|
||||
"""
|
||||
sqlsuf = ""
|
||||
if count is not None:
|
||||
sqlsuf += "LIMIT %s " % count
|
||||
if offset:
|
||||
sqlsuf += "OFFSET %s " % offset
|
||||
|
||||
c = self.mysql_conn.cursor()
|
||||
select = "SELECT domain, MAX(stored) as stored, data \
|
||||
FROM ( \
|
||||
SELECT * FROM %s ORDER BY stored DESC \
|
||||
) AS inner_ordered \
|
||||
GROUP BY domain \
|
||||
ORDER BY %s %s %s" % (self._table, order, olist, sqlsuf)
|
||||
c.execute(select)
|
||||
for domain, stored, datas in c:
|
||||
obj = pickle.loads(datas)
|
||||
if path and isinstance(obj, dict):
|
||||
yield domain, stored, stats_getpath(obj, path)
|
||||
else:
|
||||
yield domain, stored, obj
|
||||
|
||||
def domain_count(self):
|
||||
"""
|
||||
Return the number of domains stored in the database
|
||||
"""
|
||||
c = self.mysql_conn.cursor()
|
||||
c.execute("SELECT COUNT(DISTINCT(domain)) FROM %s" % self._table)
|
||||
return c.fetchone()[0]
|
||||
|
||||
def put(self, domain, data, timestamp=None):
|
||||
"""
|
||||
Store data in history using the given timestamp (or now if omitted).
|
||||
data can be any pickable object
|
||||
"""
|
||||
stored = timestamp if timestamp else datetime.now()
|
||||
datas = pickle.dumps(data)
|
||||
|
||||
c = self.mysql_conn.cursor()
|
||||
c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % self._table, (domain, stored, datas))
|
||||
self.mysql_conn.commit()
|
||||
|
||||
def remove(self, domain):
|
||||
"""
|
||||
Remove all data stored for the given domain
|
||||
"""
|
||||
c = self.mysql_conn.cursor()
|
||||
c.execute("DELETE FROM %s WHERE domain=%%s" % self._table, domain)
|
||||
self.mysql_conn.commit()
|
||||
|
||||
|
|
@ -1,17 +1,19 @@
|
|||
from unittest import TestCase, main
|
||||
from scrapy.conf import settings
|
||||
from scrapy.stats.statscollector import StatsCollector
|
||||
from scrapy.stats.collector import StatsCollector, DummyStatsCollector
|
||||
|
||||
class StatsTest(TestCase):
|
||||
def test_stats(self):
|
||||
stats = StatsCollector(enabled=False)
|
||||
# TODO: restore stats tests for new stats collector
|
||||
"""
|
||||
stats = DummyStatsCollector()
|
||||
self.assertEqual(stats, {})
|
||||
self.assertEqual(stats.getpath('anything'), None)
|
||||
self.assertEqual(stats.get_value('anything'), None)
|
||||
self.assertEqual(stats.getpath('anything', 'default'), 'default')
|
||||
stats.setpath('test', 'value')
|
||||
self.assertEqual(stats, {})
|
||||
|
||||
stats = StatsCollector(enabled=True)
|
||||
stats = StatsCollector()
|
||||
self.assertEqual(stats, {})
|
||||
self.assertEqual(stats.getpath('anything'), None)
|
||||
self.assertEqual(stats.getpath('anything', 'default'), 'default')
|
||||
|
|
@ -47,6 +49,7 @@ class StatsTest(TestCase):
|
|||
self.assertEqual(stats.getpath('one/newnode'), 1)
|
||||
stats.incpath('one/newnode', -1)
|
||||
self.assertEqual(stats.getpath('one/newnode'), 0)
|
||||
"""
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -1,95 +0,0 @@
|
|||
try:
|
||||
import MySQLdb
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from twisted.trial import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from time import sleep
|
||||
from scrapy.store.db import DomainDataHistory
|
||||
from scrapy.utils.db import mysql_connect, parse_uri, URIValidationError
|
||||
from scrapy.conf import settings
|
||||
|
||||
class MySqlTestCase(unittest.TestCase):
|
||||
test_db = settings.get('TEST_SCRAPING_DB')
|
||||
|
||||
def setUp(self):
|
||||
if not self.test_db:
|
||||
raise unittest.SkipTest("Missing TEST_SCRAPING_DB setting")
|
||||
|
||||
try:
|
||||
mysql_connect(self.test_db)
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("MySQLdb module not available")
|
||||
except MySQLdb.OperationalError:
|
||||
raise unittest.SkipTest("Test database not available at: %s" % self.test_db)
|
||||
|
||||
class ConnectionTestCase(MySqlTestCase):
|
||||
""" Test connection wrapper """
|
||||
def test_parse_uri(self):
|
||||
self.assertRaises(URIValidationError, parse_uri, [self.test_db])
|
||||
self.assertRaises(URIValidationError, parse_uri, self.test_db.replace('mysql:', 'gopher:'))
|
||||
|
||||
d = parse_uri(self.test_db)
|
||||
|
||||
self.assertTrue(isinstance(d, dict))
|
||||
self.assertTrue(all(key in d for key in ('user', 'host', 'db')))
|
||||
|
||||
def test_mysql_connect(self):
|
||||
if not self.test_db:
|
||||
raise unittest.SkipTest("Missing TEST_SCRAPING_DB setting")
|
||||
|
||||
self.assertTrue(isinstance(mysql_connect(self.test_db), MySQLdb.connection))
|
||||
self.assertTrue(isinstance(mysql_connect(parse_uri(self.test_db)), MySQLdb.connection))
|
||||
|
||||
class ProductComparisonTestCase(MySqlTestCase):
|
||||
""" Test product comparison functions """
|
||||
def test_domaindatahistory(self):
|
||||
ddh = DomainDataHistory(self.test_db, 'domain_data_history')
|
||||
c = ddh.mysql_conn.cursor()
|
||||
c.execute("DELETE FROM domain_data_history")
|
||||
ddh.mysql_conn.commit()
|
||||
|
||||
def now_nomicro():
|
||||
now = datetime.now()
|
||||
return now - timedelta(microseconds=now.microsecond)
|
||||
|
||||
assert hasattr(ddh.get('scrapy.org'), '__iter__')
|
||||
self.assertEqual(list(ddh.get('scrapy.org')), [])
|
||||
|
||||
self.assertEqual(ddh.domain_count(), 0)
|
||||
|
||||
now = now_nomicro()
|
||||
ddh.put('scrapy.org', 'value', timestamp=now)
|
||||
self.assertEqual(list(ddh.get('scrapy.org')), [(now, 'value')])
|
||||
self.assertEqual(list(ddh.get('scrapy2.org')), [])
|
||||
|
||||
sleep(1)
|
||||
now2 = now_nomicro()
|
||||
ddh.put('scrapy.org', 'newvalue', timestamp=now2)
|
||||
self.assertEqual(list(ddh.getall('scrapy.org')), [(now2, 'newvalue'), (now, 'value')])
|
||||
|
||||
self.assertEqual(ddh.getlast('scrapy.org'), (now2, 'newvalue'))
|
||||
self.assertEqual(ddh.getlast('scrapy.org', offset=1), (now, 'value'))
|
||||
self.assertEqual(ddh.getlast('scrapy2.org'), None)
|
||||
|
||||
ddh.remove('scrapy.org')
|
||||
self.assertEqual(list(ddh.get('scrapy.org')), [])
|
||||
|
||||
now3 = now_nomicro()
|
||||
d1 = {'name': 'John', 'surname': 'Doe'}
|
||||
ddh.put('scrapy.org', d1, timestamp=now3)
|
||||
self.assertEqual(list(ddh.getall('scrapy.org')), [(now3, d1)])
|
||||
|
||||
# get path support
|
||||
self.assertEqual(ddh.getlast('scrapy.org', path='name'), (now3, 'John'))
|
||||
# behaviour for non existant paths
|
||||
self.assertEqual(ddh.getlast('scrapy.org', path='name2'), (now3, None))
|
||||
|
||||
self.assertEqual(ddh.domain_count(), 1)
|
||||
|
||||
self.assertEqual(list(ddh.getlast_alldomains()),
|
||||
[('scrapy.org', now3, {'surname': 'Doe', 'name': 'John'})])
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -65,14 +65,6 @@ def memoize(cache, hash):
|
|||
return wrapper
|
||||
return decorator
|
||||
|
||||
def stats_getpath(dict_, path, default=None):
|
||||
for key in path.split('/'):
|
||||
if key in dict_:
|
||||
dict_ = dict_[key]
|
||||
else:
|
||||
return default
|
||||
return dict_
|
||||
|
||||
def load_object(path):
|
||||
"""Load an object given its absolute object path, and return it.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue