RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removed signals/stats singletons 

This change removes singletons for stats collection and signal
dispatching facilities, by making them a member of the Crawler class.

Here are some examples to illustrates the old and new API:

Signals - before:

    from scrapy import signals
    from scrapy.xlib.pydispatch import dispatcher
    dispatcher.connect(self.spider_opened, signals.spider_opened)

Signals - now:

    from scrapy import signals
    crawler.signals.connect(self.spider.opened, signals.spider_opened)

Stats collection - before:

    from scrapy.stats import stats
    stats.inc_value('foo')

Stats collection - now:

    crawler.stats.inc_value('foo')

Backwards compatibility was retained as much as possible and the old API
has been properly flagged with deprecation warnings.
This commit is contained in:
Pablo Hoffman 2012-03-06 08:59:20 -02:00
parent 36f47a4aec
commit 1e12c92b8f
33 changed files with 232 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/contrib/closespider.py
View File

@ -8,7 +8,6 @@ from collections import defaultdict
from twisted.internet import reactor
from twisted.python import log as txlog
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals, log
@ -29,12 +28,12 @@ class CloseSpider(object):
if self.errorcount:
txlog.addObserver(self.catch_log)
if self.pagecount:
dispatcher.connect(self.page_count, signal=signals.response_received)
crawler.signals.connect(self.page_count, signal=signals.response_received)
if self.timeout:
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
if self.itemcount:
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
def from_crawler(cls, crawler):

31
scrapy/contrib/corestats.py
View File

@ -3,30 +3,33 @@ Extension for collecting core stats like items scraped and start/finish times
"""
import datetime
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.stats import stats
class CoreStats(object):
def __init__(self):
dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened)
dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing)
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.item_dropped, signal=signals.item_dropped)
def __init__(self, stats):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.stats)
crawler.signals.connect(o.stats_spider_opened, signal=signals.stats_spider_opened)
crawler.signals.connect(o.stats_spider_closing, signal=signals.stats_spider_closing)
crawler.signals.connect(o.item_scraped, signal=signals.item_scraped)
crawler.signals.connect(o.item_dropped, signal=signals.item_dropped)
return o
def stats_spider_opened(self, spider):
stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
def stats_spider_closing(self, spider, reason):
stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
stats.set_value('finish_reason', reason, spider=spider)
self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
self.stats.set_value('finish_reason', reason, spider=spider)
def item_scraped(self, item, spider):
stats.inc_value('item_scraped_count', spider=spider)
self.stats.inc_value('item_scraped_count', spider=spider)
def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__
stats.inc_value('item_dropped_count', spider=spider)
stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)
self.stats.inc_value('item_dropped_count', spider=spider)
self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider)

18
scrapy/contrib/downloadermiddleware/httpcache.py
View File

@ -5,11 +5,9 @@ import cPickle as pickle
from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.http import Headers
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.stats import stats
from scrapy.responsetypes import responsetypes
from scrapy.utils.request import request_fingerprint
from scrapy.utils.httpobj import urlparse_cached
@ -19,19 +17,21 @@ from scrapy.utils.project import data_path
class HttpCacheMiddleware(object):
def __init__(self, settings):
def __init__(self, settings, stats):
if not settings.getbool('HTTPCACHE_ENABLED'):
raise NotConfigured
self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES'))
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
return cls(crawler.settings)
o = cls(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def spider_opened(self, spider):
self.storage.open_spider(spider)
@ -45,10 +45,10 @@ class HttpCacheMiddleware(object):
response = self.storage.retrieve_response(spider, request)
if response and self.is_cacheable_response(response):
response.flags.append('cached')
stats.inc_value('httpcache/hit', spider=spider)
self.stats.inc_value('httpcache/hit', spider=spider)
return response
stats.inc_value('httpcache/miss', spider=spider)
self.stats.inc_value('httpcache/miss', spider=spider)
if self.ignore_missing:
raise IgnoreRequest("Ignored request not in cache: %s" % request)
@ -57,7 +57,7 @@ class HttpCacheMiddleware(object):
and self.is_cacheable_response(response)
and 'cached' not in response.flags):
self.storage.store_response(spider, request, response)
stats.inc_value('httpcache/store', spider=spider)
self.stats.inc_value('httpcache/store', spider=spider)
return response
def is_cacheable_response(self, response):

6
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -6,8 +6,6 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
import robotparser
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals, log
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
@ -24,8 +22,8 @@ class RobotsTxtMiddleware(object):
self._parsers = {}
self._spider_netlocs = {}
self._useragents = {}
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
crawler.signals.connect(self.spider_opened, signals.spider_opened)
crawler.signals.connect(self.spider_closed, signals.spider_closed)
@classmethod
def from_crawler(cls, crawler):

22
scrapy/contrib/downloadermiddleware/stats.py
View File

@ -1,30 +1,32 @@
from scrapy.exceptions import NotConfigured
from scrapy.utils.request import request_httprepr
from scrapy.utils.response import response_httprepr
from scrapy.stats import stats
class DownloaderStats(object):
def __init__(self, stats):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('DOWNLOADER_STATS'):
raise NotConfigured
return cls()
return cls(crawler.stats)
def process_request(self, request, spider):
stats.inc_value('downloader/request_count', spider=spider)
stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider)
self.stats.inc_value('downloader/request_count', spider=spider)
self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider)
reqlen = len(request_httprepr(request))
stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider)
def process_response(self, request, response, spider):
stats.inc_value('downloader/response_count', spider=spider)
stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider)
self.stats.inc_value('downloader/response_count', spider=spider)
self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider)
reslen = len(response_httprepr(response))
stats.inc_value('downloader/response_bytes', reslen, spider=spider)
self.stats.inc_value('downloader/response_bytes', reslen, spider=spider)
return response
def process_exception(self, request, exception, spider):
ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)
stats.inc_value('downloader/exception_count', spider=spider)
stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)
self.stats.inc_value('downloader/exception_count', spider=spider)
self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)

12
scrapy/contrib/feedexport.py
View File

@ -15,7 +15,6 @@ from twisted.internet import defer, threads
from w3lib.url import file_uri_to_path
from scrapy import log, signals
from scrapy.xlib.pydispatch import dispatcher
from scrapy.utils.ftp import ftp_makedirs_cwd
from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import load_object
@ -149,9 +148,14 @@ class FeedExporter(object):
uripar = settings['FEED_URI_PARAMS']
self._uripar = load_object(uripar) if uripar else lambda x, y: None
self.slots = {}
dispatcher.connect(self.open_spider, signals.spider_opened)
dispatcher.connect(self.close_spider, signals.spider_closed)
dispatcher.connect(self.item_scraped, signals.item_scraped)
@classmethod
def from_crawler(cls, crawler):
o = cls()
crawler.signals.connect(o.open_spider, signals.spider_opened)
crawler.signals.connect(o.close_spider, signals.spider_closed)
crawler.signals.connect(o.item_scraped, signals.item_scraped)
return o
@classmethod
def from_crawler(cls, crawler):

21
scrapy/contrib/logstats.py
View File

@ -1,6 +1,5 @@
from twisted.internet import task
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import NotConfigured
from scrapy import log, signals
@ -19,12 +18,20 @@ class LogStats(object):
self.interval = interval
self.slots = {}
self.multiplier = 60.0 / self.interval
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.response_received, signal=signals.response_received)
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
dispatcher.connect(self.engine_started, signal=signals.engine_started)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
interval = settings.getfloat('LOGSTATS_INTERVAL')
if not interval:
raise NotConfigured
o = cls(interval)
crawler.signals.connect(o.item_scraped, signal=signals.item_scraped)
crawler.signals.connect(o.response_received, signal=signals.response_received)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
crawler.signals.connect(o.engine_started, signal=signals.engine_started)
crawler.signals.connect(o.engine_stopped, signal=signals.engine_stopped)
return o
@classmethod
def from_crawler(cls, crawler):

19
scrapy/contrib/memdebug.py
View File

@ -6,30 +6,29 @@ See documentation in docs/topics/extensions.rst
import gc
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.stats import stats
from scrapy.utils.trackref import live_refs
class MemoryDebugger(object):
def __init__(self, trackrefs=False):
def __init__(self, stats, trackrefs=False):
try:
import libxml2
self.libxml2 = libxml2
except ImportError:
self.libxml2 = None
self.stats = stats
self.trackrefs = trackrefs
dispatcher.connect(self.engine_started, signals.engine_started)
dispatcher.connect(self.engine_stopped, signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
if not crawler.settings.getbool('MEMDEBUG_ENABLED'):
raise NotConfigured
return cls(crawler.settings.getbool('TRACK_REFS'))
o = cls(crawler.stats, crawler.settings.getbool('TRACK_REFS'))
crawler.signals.connect(o.engine_started, signals.engine_started)
crawler.signals.connect(o.engine_stopped, signals.engine_stopped)
return o
def engine_started(self):
if self.libxml2:
@ -38,11 +37,11 @@ class MemoryDebugger(object):
def engine_stopped(self):
if self.libxml2:
self.libxml2.cleanupParser()
stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1))
self.stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1))
gc.collect()
stats.set_value('memdebug/gc_garbage_count', len(gc.garbage))
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage))
if self.trackrefs:
for cls, wdict in live_refs.iteritems():
if not wdict:
continue
stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict))
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict))

22
scrapy/contrib/memusage.py
View File

@ -9,12 +9,9 @@ from pprint import pformat
from twisted.internet import task
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy import log
from scrapy import signals, log
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.stats import stats
from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported
from scrapy.utils.engine import get_engine_status
@ -33,8 +30,8 @@ class MemoryUsage(object):
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
self.report = crawler.settings.getbool('MEMUSAGE_REPORT')
self.mail = MailSender()
dispatcher.connect(self.engine_started, signal=signals.engine_started)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
crawler.signals.connect(self.engine_started, signal=signals.engine_started)
crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
@ -44,7 +41,7 @@ class MemoryUsage(object):
return get_vmvalue_from_procfs('VmRSS')
def engine_started(self):
stats.set_value('memusage/startup', self.get_virtual_size())
self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())
self.tasks = []
tsk = task.LoopingCall(self.update)
self.tasks.append(tsk)
@ -64,18 +61,18 @@ class MemoryUsage(object):
tsk.stop()
def update(self):
stats.max_value('memusage/max', self.get_virtual_size())
self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
def _check_limit(self):
if self.get_virtual_size() > self.limit:
stats.set_value('memusage/limit_reached', 1)
self.crawler.stats.set_value('memusage/limit_reached', 1)
mem = self.limit/1024/1024
log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR)
if self.notify_mails:
subj = "%s terminated: memory usage exceeded %dM at %s" % \
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
self._send_report(self.notify_mails, subj)
stats.set_value('memusage/limit_notified', 1)
self.crawler.stats.set_value('memusage/limit_notified', 1)
open_spiders = self.crawler.engine.open_spiders
if open_spiders:
for spider in open_spiders:
@ -87,18 +84,19 @@ class MemoryUsage(object):
if self.warned: # warn only once
return
if self.get_virtual_size() > self.warning:
stats.set_value('memusage/warning_reached', 1)
self.crawler.stats.set_value('memusage/warning_reached', 1)
mem = self.warning/1024/1024
log.msg("Memory usage reached %dM" % mem, level=log.WARNING)
if self.notify_mails:
subj = "%s warning: memory usage reached %dM at %s" % \
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
self._send_report(self.notify_mails, subj)
stats.set_value('memusage/warning_notified', 1)
self.crawler.stats.set_value('memusage/warning_notified', 1)
self.warned = True
def _send_report(self, rcpts, subject):
"""send notification mail with some additional useful info"""
stats = self.crawler.stats
s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024)
s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024)
s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024)

11
scrapy/contrib/pipeline/images.py
View File

@ -15,12 +15,9 @@ from collections import defaultdict
from twisted.internet import defer, threads
from PIL import Image
from scrapy.xlib.pydispatch import dispatcher
from scrapy import log
from scrapy.stats import stats
from scrapy.utils.misc import md5sum
from scrapy.http import Request
from scrapy import signals
from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest
from scrapy.contrib.pipeline.media import MediaPipeline
@ -40,10 +37,6 @@ class FSImagesStore(object):
self.basedir = basedir
self._mkdir(self.basedir)
self.created_directories = defaultdict(set)
dispatcher.connect(self.spider_closed, signals.spider_closed)
def spider_closed(self, spider):
self.created_directories.pop(spider.name, None)
def persist_image(self, key, image, buf, info):
absolute_path = self._get_filesystem_path(key)
@ -274,8 +267,8 @@ class ImagesPipeline(MediaPipeline):
yield thumb_key, thumb_image, thumb_buf
def inc_stats(self, spider, status):
stats.inc_value('image_count', spider=spider)
stats.inc_value('image_status_count/%s' % status, spider=spider)
spider.crawler.stats.inc_value('image_count', spider=spider)
spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider)
def convert_image(self, image, size=None):
if image.format == 'PNG' and image.mode == 'RGBA':

9
scrapy/contrib/spidermiddleware/depth.py
View File

@ -19,16 +19,13 @@ class DepthMiddleware(object):
self.prio = prio
@classmethod
def from_settings(cls, settings):
def from_crawler(cls, crawler):
settings = crawler.settings
maxdepth = settings.getint('DEPTH_LIMIT')
usestats = settings.getbool('DEPTH_STATS')
verbose = settings.getbool('DEPTH_STATS_VERBOSE')
prio = settings.getint('DEPTH_PRIORITY')
if usestats:
from scrapy.stats import stats
else:
stats = None
return cls(maxdepth, stats, verbose, prio)
return cls(maxdepth, crawler.stats, verbose, prio)
def process_spider_output(self, response, result, spider):
def _filter(request):

10
scrapy/contrib/spidermiddleware/offsite.py
View File

@ -6,7 +6,6 @@ See documentation in docs/topics/spider-middleware.rst
import re
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
@ -17,8 +16,13 @@ class OffsiteMiddleware(object):
def __init__(self):
self.host_regexes = {}
self.domains_seen = {}
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
def from_crawler(cls, crawler):
o = cls()
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
def process_spider_output(self, response, result, spider):
for x in result:

5
scrapy/contrib/spiderstate.py
View File

@ -1,7 +1,6 @@
import os, cPickle as pickle
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
class SpiderState(object):
"""Store and load spider state during a scraping job"""
@ -12,8 +11,8 @@ class SpiderState(object):
@classmethod
def from_crawler(cls, crawler):
obj = cls(crawler.settings.get('JOBDIR'))
dispatcher.connect(obj.spider_closed, signal=signals.spider_closed)
dispatcher.connect(obj.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed)
crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
return obj
def spider_closed(self, spider):

19
scrapy/contrib/statsmailer.py
View File

@ -4,30 +4,29 @@ StatsMailer extension sends an email when a spider finishes scraping.
Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
"""
from scrapy.xlib.pydispatch import dispatcher
from scrapy.stats import stats
from scrapy import signals
from scrapy.mail import MailSender
from scrapy.exceptions import NotConfigured
class StatsMailer(object):
def __init__(self, recipients):
def __init__(self, stats, recipients):
self.stats = stats
self.recipients = recipients
if not self.recipients:
raise NotConfigured
dispatcher.connect(self.stats_spider_closed, signal=signals.stats_spider_closed)
@classmethod
def from_crawler(cls, crawler):
recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
return cls(recipients)
if not recipients:
raise NotConfigured
o = cls(crawler.stats, recipients)
crawler.connect(o.stats_spider_closed, signal=signals.stats_spider_closed)
return o
def stats_spider_closed(self, spider, spider_stats):
mail = MailSender()
body = "Global stats\n\n"
body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items())
body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items())
body += "\n\n%s stats\n\n" % spider.name
body += "\n".join("%-50s : %s" % i for i in spider_stats.items())
mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body)

5
scrapy/contrib/throttle.py
View File

@ -1,4 +1,3 @@
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.utils.httpobj import urlparse_cached
@ -63,8 +62,8 @@ class AutoThrottle(object):
if not settings.getbool('AUTOTHROTTLE_ENABLED'):
raise NotConfigured
self.crawler = crawler
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
dispatcher.connect(self.response_received, signal=signals.response_received)
crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(self.response_received, signal=signals.response_received)
self.START_DELAY = settings.getfloat("AUTOTHROTTLE_START_DELAY", 5.0)
self.CONCURRENCY_CHECK_PERIOD = settings.getint("AUTOTHROTTLE_CONCURRENCY_CHECK_PERIOD", 10)
self.MAX_CONCURRENCY = settings.getint("AUTOTHROTTLE_MAX_CONCURRENCY", 8)

3
scrapy/contrib/webservice/stats.py
View File

@ -1,9 +1,8 @@
from scrapy.webservice import JsonRpcResource
from scrapy.stats import stats
class StatsResource(JsonRpcResource):
ws_name = 'stats'
def __init__(self, crawler):
JsonRpcResource.__init__(self, crawler, stats)
JsonRpcResource.__init__(self, crawler, crawler.stats)

4
scrapy/core/downloader/__init__.py
View File

@ -8,7 +8,6 @@ from twisted.internet import reactor, defer
from twisted.python.failure import Failure
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.signal import send_catch_log
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy.exceptions import ScrapyDeprecationWarning
@ -68,6 +67,7 @@ class Downloader(object):
def __init__(self, crawler):
self.settings = crawler.settings
self.signals = crawler.signals
self.slots = {}
self.active = set()
self.handlers = DownloadHandlers()
@ -114,7 +114,7 @@ class Downloader(object):
def _enqueue_request(self, request, spider, slot):
def _downloaded(response):
send_catch_log(signal=signals.response_downloaded, \
self.signals.send_catch_log(signal=signals.response_downloaded, \
response=response, request=request, spider=spider)
return response

24
scrapy/core/engine.py
View File

@ -11,13 +11,11 @@ from twisted.internet import defer
from twisted.python.failure import Failure
from scrapy import log, signals
from scrapy.stats import stats
from scrapy.core.downloader import Downloader
from scrapy.core.scraper import Scraper
from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
from scrapy.http import Response, Request
from scrapy.utils.misc import load_object
from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
from scrapy.utils.reactor import CallLaterOnce
@ -53,7 +51,9 @@ class Slot(object):
class ExecutionEngine(object):
def __init__(self, crawler, spider_closed_callback):
self.crawler = crawler
self.settings = crawler.settings
self.signals = crawler.signals
self.slots = {}
self.running = False
self.paused = False
@ -71,7 +71,7 @@ class ExecutionEngine(object):
"""Start the execution engine"""
assert not self.running, "Engine already running"
self.start_time = time()
yield send_catch_log_deferred(signal=signals.engine_started)
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
def stop(self):
@ -195,7 +195,7 @@ class ExecutionEngine(object):
response.request = request # tie request to response received
log.msg(log.formatter.crawled(request, response, spider), \
level=log.DEBUG, spider=spider)
send_catch_log(signal=signals.response_received, \
self.signals.send_catch_log(signal=signals.response_received, \
response=response, request=request, spider=spider)
return response
@ -218,13 +218,13 @@ class ExecutionEngine(object):
spider.name
log.msg("Spider opened", spider=spider)
nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_settings(self.settings)
scheduler = self.scheduler_cls.from_crawler(self.crawler)
slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler)
self.slots[spider] = slot
yield scheduler.open(spider)
yield self.scraper.open_spider(spider)
stats.open_spider(spider)
yield send_catch_log_deferred(signals.spider_opened, spider=spider)
self.crawler.stats.open_spider(spider)
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
slot.nextcall.schedule()
def _spider_idle(self, spider):
@ -235,7 +235,7 @@ class ExecutionEngine(object):
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
res = send_catch_log(signal=signals.spider_idle, \
res = self.signals.send_catch_log(signal=signals.spider_idle, \
spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
for _, x in res):
@ -261,11 +261,11 @@ class ExecutionEngine(object):
dfd.addBoth(lambda _: slot.scheduler.close(reason))
dfd.addErrback(log.err, spider=spider)
dfd.addBoth(lambda _: send_catch_log_deferred(signal=signals.spider_closed, \
dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(signal=signals.spider_closed, \
spider=spider, reason=reason))
dfd.addErrback(log.err, spider=spider)
dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason))
dfd.addErrback(log.err, spider=spider)
dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))
@ -284,5 +284,5 @@ class ExecutionEngine(object):
@defer.inlineCallbacks
def _finish_stopping_engine(self):
yield send_catch_log_deferred(signal=signals.engine_stopped)
yield stats.engine_stopped()
yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
yield self.crawler.stats.engine_stopped()

15
scrapy/core/scheduler.py
View File

@ -6,26 +6,27 @@ from scrapy.utils.pqueue import PriorityQueue
from scrapy.utils.reqser import request_to_dict, request_from_dict
from scrapy.utils.misc import load_object
from scrapy.utils.job import job_dir
from scrapy.stats import stats
from scrapy import log
class Scheduler(object):
def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False):
def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None):
self.df = dupefilter
self.dqdir = self._dqdir(jobdir)
self.dqclass = dqclass
self.mqclass = mqclass
self.logunser = logunser
self.stats = stats
@classmethod
def from_settings(cls, settings):
def from_crawler(cls, crawler):
settings = crawler.settings
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
dupefilter = dupefilter_cls.from_settings(settings)
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')
return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser)
return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser, crawler.stats)
def has_pending_requests(self):
return len(self) > 0
@ -67,11 +68,13 @@ class Scheduler(object):
(request, str(e)), level=log.ERROR, spider=self.spider)
return
else:
stats.inc_value('scheduler/disk_enqueued', spider=self.spider)
if self.stats:
self.stats.inc_value('scheduler/disk_enqueued', spider=self.spider)
return True
def _mqpush(self, request):
stats.inc_value('scheduler/memory_enqueued', spider=self.spider)
if self.stats:
self.stats.inc_value('scheduler/memory_enqueued', spider=self.spider)
self.mqs.push(request, -request.priority)
def _dqpop(self):

13
scrapy/core/scraper.py
View File

@ -9,14 +9,12 @@ from twisted.internet import defer
from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.misc import load_object
from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
from scrapy.exceptions import CloseSpider, DropItem
from scrapy import signals
from scrapy.http import Request, Response
from scrapy.item import BaseItem
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy import log
from scrapy.stats import stats
class Slot(object):
@ -68,6 +66,7 @@ class Scraper(object):
self.itemproc = itemproc_cls.from_crawler(crawler)
self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS')
self.crawler = crawler
self.signals = crawler.signals
@defer.inlineCallbacks
def open_spider(self, spider):
@ -146,9 +145,9 @@ class Scraper(object):
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
return
log.err(_failure, "Spider error processing %s" % request, spider=spider)
send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \
self.signals.send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \
spider=spider)
stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \
self.crawler.stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \
spider=spider)
def handle_spider_output(self, result, request, response, spider):
@ -164,7 +163,7 @@ class Scraper(object):
from the given spider
"""
if isinstance(output, Request):
send_catch_log(signal=signals.request_received, request=output, \
self.signals.send_catch_log(signal=signals.request_received, request=output, \
spider=spider)
self.crawler.engine.crawl(request=output, spider=spider)
elif isinstance(output, BaseItem):
@ -199,13 +198,13 @@ class Scraper(object):
if isinstance(ex, DropItem):
log.msg(log.formatter.dropped(item, ex, response, spider), \
level=log.WARNING, spider=spider)
return send_catch_log_deferred(signal=signals.item_dropped, \
return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \
item=item, spider=spider, exception=output.value)
else:
log.err(output, 'Error processing %s' % item, spider=spider)
else:
log.msg(log.formatter.scraped(output, response, spider), \
log.DEBUG, spider=spider)
return send_catch_log_deferred(signal=signals.item_scraped, \
return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \
item=output, response=response, spider=spider)

6
scrapy/crawler.py
View File

@ -2,10 +2,10 @@ import signal
from twisted.internet import reactor, defer
from scrapy.xlib.pydispatch import dispatcher
from scrapy.core.engine import ExecutionEngine
from scrapy.resolver import CachingThreadedResolver
from scrapy.extension import ExtensionManager
from scrapy.signalmanager import SignalManager
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.misc import load_object
from scrapy import log, signals
@ -16,6 +16,8 @@ class Crawler(object):
def __init__(self, settings):
self.configured = False
self.settings = settings
self.signals = SignalManager(self)
self.stats = load_object(settings['STATS_CLASS'])(self)
def install(self):
import scrapy.project
@ -65,7 +67,7 @@ class CrawlerProcess(Crawler):
def __init__(self, *a, **kw):
super(CrawlerProcess, self).__init__(*a, **kw)
dispatcher.connect(self.stop, signals.engine_stopped)
self.signals.connect(self.stop, signals.engine_stopped)
install_shutdown_handlers(self._signal_shutdown)
def start(self):

7
scrapy/mail.py
View File

@ -17,7 +17,6 @@ from twisted.mail.smtp import ESMTPSenderFactory
from scrapy import log
from scrapy.exceptions import NotConfigured
from scrapy.conf import settings
from scrapy.utils.signal import send_catch_log
# signal sent when message is sent
@ -28,13 +27,14 @@ mail_sent = object()
class MailSender(object):
def __init__(self, smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, \
smtpport=None, debug=False):
smtpport=None, debug=False, crawler=None):
self.smtphost = smtphost or settings['MAIL_HOST']
self.smtpport = smtpport or settings.getint('MAIL_PORT')
self.smtpuser = smtpuser or settings['MAIL_USER']
self.smtppass = smtppass or settings['MAIL_PASS']
self.mailfrom = mailfrom or settings['MAIL_FROM']
self.debug = debug
self.signals = crawler.signals if crawler else None
if not self.smtphost or not self.mailfrom:
raise NotConfigured("MAIL_HOST and MAIL_FROM settings are required")
@ -65,7 +65,8 @@ class MailSender(object):
else:
msg.set_payload(body)
send_catch_log(signal=mail_sent, to=to, subject=subject, body=body,
if self.signals:
self.signals.send_catch_log(signal=mail_sent, to=to, subject=subject, body=body,
cc=cc, attach=attachs, msg=msg)
if self.debug:

27
scrapy/signalmanager.py Normal file
View File

@ -0,0 +1,27 @@
from scrapy.xlib.pydispatch import dispatcher
from scrapy.utils import signal
class SignalManager(object):
def __init__(self, sender=dispatcher.Anonymous):
self.sender = sender
def connect(self, *a, **kw):
kw.setdefault('sender', self.sender)
return dispatcher.connect(*a, **kw)
def disconnect(self, *a, **kw):
kw.setdefault('sender', self.sender)
return dispatcher.disconnect(*a, **kw)
def send_catch_log(self, *a, **kw):
kw.setdefault('sender', self.sender)
return signal.send_catch_log(*a, **kw)
def send_catch_log_deferred(self, *a, **kw):
kw.setdefault('sender', self.sender)
return signal.send_catch_log_deferred(*a, **kw)
def disconnect_all(self, *a, **kw):
kw.setdefault('sender', self.sender)
return signal.disconnect_all(*a, **kw)

6
scrapy/spidermanager.py
View File

@ -9,7 +9,6 @@ from scrapy import signals
from scrapy.interfaces import ISpiderManager
from scrapy.utils.misc import walk_modules
from scrapy.utils.spider import iter_spider_classes
from scrapy.xlib.pydispatch import dispatcher
class SpiderManager(object):
@ -22,7 +21,6 @@ class SpiderManager(object):
for name in self.spider_modules:
for module in walk_modules(name):
self._load_spiders(module)
dispatcher.connect(self.close_spider, signals.spider_closed)
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
@ -34,7 +32,9 @@ class SpiderManager(object):
@classmethod
def from_crawler(cls, crawler):
return cls.from_settings(crawler.settings)
sm = cls.from_settings(crawler.settings)
crawler.signals.connect(sm.close_spider, signals.spider_closed)
return sm
def create(self, spider_name, **spider_kwargs):
try:

14
scrapy/stats.py
View File

@ -1,9 +1,7 @@
from scrapy.statscol import DummyStatsCollector
from scrapy.conf import settings
from scrapy.utils.misc import load_object
from scrapy.project import crawler
stats = crawler.stats
# if stats are disabled use a DummyStatsCollector to improve performance
if settings.getbool('STATS_ENABLED'):
stats = load_object(settings['STATS_CLASS'])()
else:
stats = DummyStatsCollector()
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead",
ScrapyDeprecationWarning, stacklevel=2)

20
scrapy/statscol.py
View File

@ -3,20 +3,16 @@ Scrapy extension for collecting scraping stats
"""
import pprint
from scrapy.xlib.pydispatch import dispatcher
from scrapy.signals import stats_spider_opened, stats_spider_closing, \
stats_spider_closed
from scrapy.utils.signal import send_catch_log
from scrapy import signals
from scrapy import log
from scrapy.conf import settings
class StatsCollector(object):
def __init__(self):
self._dump = settings.getbool('STATS_DUMP')
def __init__(self, crawler):
self._dump = crawler.settings.getbool('STATS_DUMP')
self._stats = {None: {}} # None is for global stats
self._signals = crawler.signals
def get_value(self, key, default=None, spider=None):
return self._stats[spider].get(key, default)
@ -50,12 +46,12 @@ class StatsCollector(object):
def open_spider(self, spider):
self._stats[spider] = {}
send_catch_log(stats_spider_opened, spider=spider)
self._signals.send_catch_log(stats_spider_opened, spider=spider)
def close_spider(self, spider, reason):
send_catch_log(stats_spider_closing, spider=spider, reason=reason)
self._signals.send_catch_log(stats_spider_closing, spider=spider, reason=reason)
stats = self._stats.pop(spider)
send_catch_log(stats_spider_closed, spider=spider, reason=reason, \
self._signals.send_catch_log(stats_spider_closed, spider=spider, reason=reason, \
spider_stats=stats)
if self._dump:
log.msg("Dumping spider stats:\n" + pprint.pformat(stats), \
@ -73,8 +69,8 @@ class StatsCollector(object):
class MemoryStatsCollector(StatsCollector):
def __init__(self):
super(MemoryStatsCollector, self).__init__()
def __init__(self, crawler):
super(MemoryStatsCollector, self).__init__(crawler)
self.spider_stats = {}
def _persist_stats(self, stats, spider=None):

11
scrapy/telnet.py
View File

@ -10,11 +10,8 @@ from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
from twisted.internet import protocol
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import NotConfigured
from scrapy.stats import stats
from scrapy import log, signals
from scrapy.utils.signal import send_catch_log
from scrapy.utils.trackref import print_live_refs
from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
@ -39,8 +36,8 @@ class TelnetConsole(protocol.ServerFactory):
self.noisy = False
self.portrange = map(int, crawler.settings.getlist('TELNETCONSOLE_PORT'))
self.host = crawler.settings['TELNETCONSOLE_HOST']
dispatcher.connect(self.start_listening, signals.engine_started)
dispatcher.connect(self.stop_listening, signals.engine_stopped)
self.crawler.signals.connect(self.start_listening, signals.engine_started)
self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
@ -70,7 +67,7 @@ class TelnetConsole(protocol.ServerFactory):
'slot': slot,
'manager': self.crawler,
'extensions': self.crawler.extensions,
'stats': stats,
'stats': self.crawler.stats,
'spiders': self.crawler.spiders,
'settings': self.crawler.settings,
'est': lambda: print_engine_status(self.crawler.engine),
@ -80,5 +77,5 @@ class TelnetConsole(protocol.ServerFactory):
'help': "This is Scrapy telnet console. For more info see: " \
"http://doc.scrapy.org/en/latest/topics/telnetconsole.html",
}
send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars)
return telnet_vars

13
scrapy/tests/test_downloadermiddleware_httpcache.py
View File

@ -3,9 +3,9 @@ import unittest, tempfile, shutil, time
from scrapy.http import Response, HtmlResponse, Request
from scrapy.spider import BaseSpider
from scrapy.contrib.downloadermiddleware.httpcache import FilesystemCacheStorage, HttpCacheMiddleware
from scrapy.stats import stats
from scrapy.settings import Settings
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.test import get_crawler
class HttpCacheMiddlewareTest(unittest.TestCase):
@ -13,14 +13,15 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
storage_class = FilesystemCacheStorage
def setUp(self):
self.crawler = get_crawler()
self.spider = BaseSpider('example.com')
self.tmpdir = tempfile.mkdtemp()
self.request = Request('http://www.example.com', headers={'User-Agent': 'test'})
self.response = Response('http://www.example.com', headers={'Content-Type': 'text/html'}, body='test body', status=202)
stats.open_spider(self.spider)
self.crawler.stats.open_spider(self.spider)
def tearDown(self):
stats.close_spider(self.spider, '')
self.crawler.stats.close_spider(self.spider, '')
shutil.rmtree(self.tmpdir)
def _get_settings(self, **new_settings):
@ -37,7 +38,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
return self.storage_class(self._get_settings(**new_settings))
def _get_middleware(self, **new_settings):
return HttpCacheMiddleware(self._get_settings(**new_settings))
return HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats)
def test_storage(self):
storage = self._get_storage()
@ -58,7 +59,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
assert storage.retrieve_response(self.spider, self.request)
def test_middleware(self):
mw = HttpCacheMiddleware(self._get_settings())
mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
assert mw.process_request(self.request, self.spider) is None
mw.process_response(self.request, self.response, self.spider)
response = mw.process_request(self.request, self.spider)
@ -67,7 +68,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
assert 'cached' in response.flags
def test_different_request_response_urls(self):
mw = HttpCacheMiddleware(self._get_settings())
mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
req = Request('http://host.com/path')
res = Response('http://host2.net/test.html')
assert mw.process_request(req, self.spider) is None

15
scrapy/tests/test_downloadermiddleware_stats.py
View File

@ -3,35 +3,36 @@ from unittest import TestCase
from scrapy.contrib.downloadermiddleware.stats import DownloaderStats
from scrapy.http import Request, Response
from scrapy.spider import BaseSpider
from scrapy.stats import stats
from scrapy.utils.test import get_crawler
class TestDownloaderStats(TestCase):
def setUp(self):
self.crawler = get_crawler()
self.spider = BaseSpider('scrapytest.org')
self.mw = DownloaderStats()
self.mw = DownloaderStats(self.crawler.stats)
stats.open_spider(self.spider)
self.crawler.stats.open_spider(self.spider)
self.req = Request('http://scrapytest.org')
self.res = Response('scrapytest.org', status=400)
def test_process_request(self):
self.mw.process_request(self.req, self.spider)
self.assertEqual(stats.get_value('downloader/request_count', \
self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \
spider=self.spider), 1)
def test_process_response(self):
self.mw.process_response(self.req, self.res, self.spider)
self.assertEqual(stats.get_value('downloader/response_count', \
self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \
spider=self.spider), 1)
def test_process_exception(self):
self.mw.process_exception(self.req, Exception(), self.spider)
self.assertEqual(stats.get_value('downloader/exception_count', \
self.assertEqual(self.crawler.stats.get_value('downloader/exception_count', \
spider=self.spider), 1)
def tearDown(self):
stats.close_spider(self.spider, '')
self.crawler.stats.close_spider(self.spider, '')

12
scrapy/tests/test_mail.py
View File

@ -1,22 +1,22 @@
from cStringIO import StringIO
import unittest
from scrapy.xlib.pydispatch import dispatcher
from scrapy.mail import MailSender, mail_sent
from scrapy.utils.test import get_crawler
class MailSenderTest(unittest.TestCase):
def setUp(self):
self.catched_msg = None
dispatcher.connect(self._catch_mail_sent, signal=mail_sent)
self.crawler = get_crawler()
self.crawler.signals.connect(self._catch_mail_sent, signal=mail_sent)
def tearDown(self):
dispatcher.disconnect(self._catch_mail_sent, signal=mail_sent)
self.crawler.signals.disconnect(self._catch_mail_sent, signal=mail_sent)
def test_send(self):
mailsender = MailSender(debug=True)
mailsender = MailSender(debug=True, crawler=self.crawler)
mailsender.send(to=['test@scrapy.org'], subject='subject', body='body')
assert self.catched_msg
@ -36,7 +36,7 @@ class MailSenderTest(unittest.TestCase):
attach.seek(0)
attachs = [('attachment', 'text/plain', attach)]
mailsender = MailSender(debug=True)
mailsender = MailSender(debug=True, crawler=self.crawler)
mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
attachs=attachs)

3
scrapy/tests/test_spidermiddleware_depth.py
View File

@ -4,6 +4,7 @@ from scrapy.contrib.spidermiddleware.depth import DepthMiddleware
from scrapy.http import Response, Request
from scrapy.spider import BaseSpider
from scrapy.statscol import StatsCollector
from scrapy.utils.test import get_crawler
class TestDepthMiddleware(TestCase):
@ -11,7 +12,7 @@ class TestDepthMiddleware(TestCase):
def setUp(self):
self.spider = BaseSpider('scrapytest.org')
self.stats = StatsCollector()
self.stats = StatsCollector(get_crawler())
self.stats.open_spider(self.spider)
self.mw = DepthMiddleware(1, self.stats, True)

21
scrapy/tests/test_stats.py
View File

@ -1,18 +1,19 @@
import unittest
from scrapy.spider import BaseSpider
from scrapy.xlib.pydispatch import dispatcher
from scrapy.statscol import StatsCollector, DummyStatsCollector
from scrapy.signals import stats_spider_opened, stats_spider_closing, \
stats_spider_closed
from scrapy.utils.test import get_crawler
class StatsCollectorTest(unittest.TestCase):
def setUp(self):
self.crawler = get_crawler()
self.spider = BaseSpider('foo')
def test_collector(self):
stats = StatsCollector()
stats = StatsCollector(self.crawler)
self.assertEqual(stats.get_stats(), {})
self.assertEqual(stats.get_value('anything'), None)
self.assertEqual(stats.get_value('anything', 'default'), 'default')
@ -39,7 +40,7 @@ class StatsCollectorTest(unittest.TestCase):
self.assertEqual(stats.get_value('test4'), 7)
def test_dummy_collector(self):
stats = DummyStatsCollector()
stats = DummyStatsCollector(self.crawler)
self.assertEqual(stats.get_stats(), {})
self.assertEqual(stats.get_value('anything'), None)
self.assertEqual(stats.get_value('anything', 'default'), 'default')
@ -70,11 +71,11 @@ class StatsCollectorTest(unittest.TestCase):
assert spider_stats == {'test': 1}
signals_catched.add(stats_spider_closed)
dispatcher.connect(spider_opened, signal=stats_spider_opened)
dispatcher.connect(spider_closing, signal=stats_spider_closing)
dispatcher.connect(spider_closed, signal=stats_spider_closed)
self.crawler.signals.connect(spider_opened, signal=stats_spider_opened)
self.crawler.signals.connect(spider_closing, signal=stats_spider_closing)
self.crawler.signals.connect(spider_closed, signal=stats_spider_closed)
stats = StatsCollector()
stats = StatsCollector(self.crawler)
stats.open_spider(self.spider)
stats.set_value('test', 1, spider=self.spider)
self.assertEqual([(self.spider, {'test': 1})], list(stats.iter_spider_stats()))
@ -83,9 +84,9 @@ class StatsCollectorTest(unittest.TestCase):
assert stats_spider_closing in signals_catched
assert stats_spider_closed in signals_catched
dispatcher.disconnect(spider_opened, signal=stats_spider_opened)
dispatcher.disconnect(spider_closing, signal=stats_spider_closing)
dispatcher.disconnect(spider_closed, signal=stats_spider_closed)
self.crawler.signals.disconnect(spider_opened, signal=stats_spider_opened)
self.crawler.signals.disconnect(spider_closing, signal=stats_spider_closing)
self.crawler.signals.disconnect(spider_closed, signal=stats_spider_closed)
if __name__ == "__main__":
unittest.main()

5
scrapy/webservice.py
View File

@ -6,7 +6,6 @@ See docs/topics/webservice.rst
from twisted.web import server, error
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import NotConfigured
from scrapy import log, signals
from scrapy.utils.jsonrpc import jsonrpc_server_call
@ -80,8 +79,8 @@ class WebService(server.Site):
root.putChild(res.ws_name, res)
server.Site.__init__(self, root, logPath=logfile)
self.noisy = False
dispatcher.connect(self.start_listening, signals.engine_started)
dispatcher.connect(self.stop_listening, signals.engine_stopped)
crawler.signals.connect(self.start_listening, signals.engine_started)
crawler.signals.connect(self.stop_listening, signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):