diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index f4611f4fc..d9b01e689 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -8,7 +8,6 @@ from collections import defaultdict from twisted.internet import reactor from twisted.python import log as txlog -from scrapy.xlib.pydispatch import dispatcher from scrapy import signals, log @@ -29,12 +28,12 @@ class CloseSpider(object): if self.errorcount: txlog.addObserver(self.catch_log) if self.pagecount: - dispatcher.connect(self.page_count, signal=signals.response_received) + crawler.signals.connect(self.page_count, signal=signals.response_received) if self.timeout: - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) if self.itemcount: - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py index da25d6a97..f45e78a43 100644 --- a/scrapy/contrib/corestats.py +++ b/scrapy/contrib/corestats.py @@ -3,30 +3,33 @@ Extension for collecting core stats like items scraped and start/finish times """ import datetime -from scrapy.xlib.pydispatch import dispatcher - from scrapy import signals -from scrapy.stats import stats class CoreStats(object): - def __init__(self): - dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened) - dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing) - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.item_dropped, signal=signals.item_dropped) + def __init__(self, stats): + self.stats = stats + + @classmethod + def from_crawler(cls, crawler): + o = cls(crawler.stats) + crawler.signals.connect(o.stats_spider_opened, signal=signals.stats_spider_opened) + crawler.signals.connect(o.stats_spider_closing, signal=signals.stats_spider_closing) + crawler.signals.connect(o.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(o.item_dropped, signal=signals.item_dropped) + return o def stats_spider_opened(self, spider): - stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) + self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) def stats_spider_closing(self, spider, reason): - stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) - stats.set_value('finish_reason', reason, spider=spider) + self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) + self.stats.set_value('finish_reason', reason, spider=spider) def item_scraped(self, item, spider): - stats.inc_value('item_scraped_count', spider=spider) + self.stats.inc_value('item_scraped_count', spider=spider) def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ - stats.inc_value('item_dropped_count', spider=spider) - stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) + self.stats.inc_value('item_dropped_count', spider=spider) + self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 642f9bda7..cb12ac031 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -5,11 +5,9 @@ import cPickle as pickle from w3lib.http import headers_dict_to_raw, headers_raw_to_dict -from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.http import Headers from scrapy.exceptions import NotConfigured, IgnoreRequest -from scrapy.stats import stats from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.httpobj import urlparse_cached @@ -19,19 +17,21 @@ from scrapy.utils.project import data_path class HttpCacheMiddleware(object): - def __init__(self, settings): + def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + self.stats = stats @classmethod def from_crawler(cls, crawler): - return cls(crawler.settings) + o = cls(crawler.settings, crawler.stats) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o def spider_opened(self, spider): self.storage.open_spider(spider) @@ -45,10 +45,10 @@ class HttpCacheMiddleware(object): response = self.storage.retrieve_response(spider, request) if response and self.is_cacheable_response(response): response.flags.append('cached') - stats.inc_value('httpcache/hit', spider=spider) + self.stats.inc_value('httpcache/hit', spider=spider) return response - stats.inc_value('httpcache/miss', spider=spider) + self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) @@ -57,7 +57,7 @@ class HttpCacheMiddleware(object): and self.is_cacheable_response(response) and 'cached' not in response.flags): self.storage.store_response(spider, request, response) - stats.inc_value('httpcache/store', spider=spider) + self.stats.inc_value('httpcache/store', spider=spider) return response def is_cacheable_response(self, response): diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index bf64677f8..8e9feda81 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -6,8 +6,6 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. import robotparser -from scrapy.xlib.pydispatch import dispatcher - from scrapy import signals, log from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request @@ -24,8 +22,8 @@ class RobotsTxtMiddleware(object): self._parsers = {} self._spider_netlocs = {} self._useragents = {} - dispatcher.connect(self.spider_opened, signals.spider_opened) - dispatcher.connect(self.spider_closed, signals.spider_closed) + crawler.signals.connect(self.spider_opened, signals.spider_opened) + crawler.signals.connect(self.spider_closed, signals.spider_closed) @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py index 32bad0bb3..9c0ad90a5 100644 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ b/scrapy/contrib/downloadermiddleware/stats.py @@ -1,30 +1,32 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.request import request_httprepr from scrapy.utils.response import response_httprepr -from scrapy.stats import stats class DownloaderStats(object): + def __init__(self, stats): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('DOWNLOADER_STATS'): raise NotConfigured - return cls() + return cls(crawler.stats) def process_request(self, request, spider): - stats.inc_value('downloader/request_count', spider=spider) - stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) + self.stats.inc_value('downloader/request_count', spider=spider) + self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) reqlen = len(request_httprepr(request)) - stats.inc_value('downloader/request_bytes', reqlen, spider=spider) + self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) def process_response(self, request, response, spider): - stats.inc_value('downloader/response_count', spider=spider) - stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) + self.stats.inc_value('downloader/response_count', spider=spider) + self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) reslen = len(response_httprepr(response)) - stats.inc_value('downloader/response_bytes', reslen, spider=spider) + self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response def process_exception(self, request, exception, spider): ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) - stats.inc_value('downloader/exception_count', spider=spider) - stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) + self.stats.inc_value('downloader/exception_count', spider=spider) + self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index cdda804f7..fd84c59a1 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -15,7 +15,6 @@ from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from scrapy import log, signals -from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.ftp import ftp_makedirs_cwd from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object @@ -149,9 +148,14 @@ class FeedExporter(object): uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None self.slots = {} - dispatcher.connect(self.open_spider, signals.spider_opened) - dispatcher.connect(self.close_spider, signals.spider_closed) - dispatcher.connect(self.item_scraped, signals.item_scraped) + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.open_spider, signals.spider_opened) + crawler.signals.connect(o.close_spider, signals.spider_closed) + crawler.signals.connect(o.item_scraped, signals.item_scraped) + return o @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py index a22c1cb2a..47b762b07 100644 --- a/scrapy/contrib/logstats.py +++ b/scrapy/contrib/logstats.py @@ -1,6 +1,5 @@ from twisted.internet import task -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy import log, signals @@ -19,12 +18,20 @@ class LogStats(object): self.interval = interval self.slots = {} self.multiplier = 60.0 / self.interval - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.response_received, signal=signals.response_received) - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - dispatcher.connect(self.engine_started, signal=signals.engine_started) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) + + @classmethod + def from_crawler(cls, crawler): + interval = settings.getfloat('LOGSTATS_INTERVAL') + if not interval: + raise NotConfigured + o = cls(interval) + crawler.signals.connect(o.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(o.response_received, signal=signals.response_received) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + crawler.signals.connect(o.engine_started, signal=signals.engine_started) + crawler.signals.connect(o.engine_stopped, signal=signals.engine_stopped) + return o @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index cb9051fd2..8965a1ebd 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -6,30 +6,29 @@ See documentation in docs/topics/extensions.rst import gc -from scrapy.xlib.pydispatch import dispatcher - from scrapy import signals from scrapy.exceptions import NotConfigured -from scrapy.stats import stats from scrapy.utils.trackref import live_refs class MemoryDebugger(object): - def __init__(self, trackrefs=False): + def __init__(self, stats, trackrefs=False): try: import libxml2 self.libxml2 = libxml2 except ImportError: self.libxml2 = None + self.stats = stats self.trackrefs = trackrefs - dispatcher.connect(self.engine_started, signals.engine_started) - dispatcher.connect(self.engine_stopped, signals.engine_stopped) @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('MEMDEBUG_ENABLED'): raise NotConfigured - return cls(crawler.settings.getbool('TRACK_REFS')) + o = cls(crawler.stats, crawler.settings.getbool('TRACK_REFS')) + crawler.signals.connect(o.engine_started, signals.engine_started) + crawler.signals.connect(o.engine_stopped, signals.engine_stopped) + return o def engine_started(self): if self.libxml2: @@ -38,11 +37,11 @@ class MemoryDebugger(object): def engine_stopped(self): if self.libxml2: self.libxml2.cleanupParser() - stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) + self.stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) gc.collect() - stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) + self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) if self.trackrefs: for cls, wdict in live_refs.iteritems(): if not wdict: continue - stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict)) + self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict)) diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 23ef6ec8e..6388c42af 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -9,12 +9,9 @@ from pprint import pformat from twisted.internet import task -from scrapy.xlib.pydispatch import dispatcher -from scrapy import signals -from scrapy import log +from scrapy import signals, log from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.stats import stats from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported from scrapy.utils.engine import get_engine_status @@ -33,8 +30,8 @@ class MemoryUsage(object): self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 self.report = crawler.settings.getbool('MEMUSAGE_REPORT') self.mail = MailSender() - dispatcher.connect(self.engine_started, signal=signals.engine_started) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) + crawler.signals.connect(self.engine_started, signal=signals.engine_started) + crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @classmethod def from_crawler(cls, crawler): @@ -44,7 +41,7 @@ class MemoryUsage(object): return get_vmvalue_from_procfs('VmRSS') def engine_started(self): - stats.set_value('memusage/startup', self.get_virtual_size()) + self.crawler.stats.set_value('memusage/startup', self.get_virtual_size()) self.tasks = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) @@ -64,18 +61,18 @@ class MemoryUsage(object): tsk.stop() def update(self): - stats.max_value('memusage/max', self.get_virtual_size()) + self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): if self.get_virtual_size() > self.limit: - stats.set_value('memusage/limit_reached', 1) + self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit/1024/1024 log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR) if self.notify_mails: subj = "%s terminated: memory usage exceeded %dM at %s" % \ (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) - stats.set_value('memusage/limit_notified', 1) + self.crawler.stats.set_value('memusage/limit_notified', 1) open_spiders = self.crawler.engine.open_spiders if open_spiders: for spider in open_spiders: @@ -87,18 +84,19 @@ class MemoryUsage(object): if self.warned: # warn only once return if self.get_virtual_size() > self.warning: - stats.set_value('memusage/warning_reached', 1) + self.crawler.stats.set_value('memusage/warning_reached', 1) mem = self.warning/1024/1024 log.msg("Memory usage reached %dM" % mem, level=log.WARNING) if self.notify_mails: subj = "%s warning: memory usage reached %dM at %s" % \ (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) - stats.set_value('memusage/warning_notified', 1) + self.crawler.stats.set_value('memusage/warning_notified', 1) self.warned = True def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" + stats = self.crawler.stats s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 29a5910e7..90ab80c7e 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -15,12 +15,9 @@ from collections import defaultdict from twisted.internet import defer, threads from PIL import Image -from scrapy.xlib.pydispatch import dispatcher from scrapy import log -from scrapy.stats import stats from scrapy.utils.misc import md5sum from scrapy.http import Request -from scrapy import signals from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest from scrapy.contrib.pipeline.media import MediaPipeline @@ -40,10 +37,6 @@ class FSImagesStore(object): self.basedir = basedir self._mkdir(self.basedir) self.created_directories = defaultdict(set) - dispatcher.connect(self.spider_closed, signals.spider_closed) - - def spider_closed(self, spider): - self.created_directories.pop(spider.name, None) def persist_image(self, key, image, buf, info): absolute_path = self._get_filesystem_path(key) @@ -274,8 +267,8 @@ class ImagesPipeline(MediaPipeline): yield thumb_key, thumb_image, thumb_buf def inc_stats(self, spider, status): - stats.inc_value('image_count', spider=spider) - stats.inc_value('image_status_count/%s' % status, spider=spider) + spider.crawler.stats.inc_value('image_count', spider=spider) + spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider) def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index 4023b4009..438051cb5 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -19,16 +19,13 @@ class DepthMiddleware(object): self.prio = prio @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings maxdepth = settings.getint('DEPTH_LIMIT') usestats = settings.getbool('DEPTH_STATS') verbose = settings.getbool('DEPTH_STATS_VERBOSE') prio = settings.getint('DEPTH_PRIORITY') - if usestats: - from scrapy.stats import stats - else: - stats = None - return cls(maxdepth, stats, verbose, prio) + return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): def _filter(request): diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index acb616043..2010ca571 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -6,7 +6,6 @@ See documentation in docs/topics/spider-middleware.rst import re -from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -17,8 +16,13 @@ class OffsiteMiddleware(object): def __init__(self): self.host_regexes = {} self.domains_seen = {} - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o def process_spider_output(self, response, result, spider): for x in result: diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py index 0347ccbbb..edf0c2946 100644 --- a/scrapy/contrib/spiderstate.py +++ b/scrapy/contrib/spiderstate.py @@ -1,7 +1,6 @@ import os, cPickle as pickle from scrapy import signals -from scrapy.xlib.pydispatch import dispatcher class SpiderState(object): """Store and load spider state during a scraping job""" @@ -12,8 +11,8 @@ class SpiderState(object): @classmethod def from_crawler(cls, crawler): obj = cls(crawler.settings.get('JOBDIR')) - dispatcher.connect(obj.spider_closed, signal=signals.spider_closed) - dispatcher.connect(obj.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed) + crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj def spider_closed(self, spider): diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index 9dc5161ec..17ad2f377 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -4,30 +4,29 @@ StatsMailer extension sends an email when a spider finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -from scrapy.xlib.pydispatch import dispatcher - -from scrapy.stats import stats from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured class StatsMailer(object): - def __init__(self, recipients): + def __init__(self, stats, recipients): + self.stats = stats self.recipients = recipients - if not self.recipients: - raise NotConfigured - dispatcher.connect(self.stats_spider_closed, signal=signals.stats_spider_closed) @classmethod def from_crawler(cls, crawler): recipients = crawler.settings.getlist("STATSMAILER_RCPTS") - return cls(recipients) - + if not recipients: + raise NotConfigured + o = cls(crawler.stats, recipients) + crawler.connect(o.stats_spider_closed, signal=signals.stats_spider_closed) + return o + def stats_spider_closed(self, spider, spider_stats): mail = MailSender() body = "Global stats\n\n" - body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items()) + body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items()) body += "\n\n%s stats\n\n" % spider.name body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py index ba462a55d..22a47aa2a 100644 --- a/scrapy/contrib/throttle.py +++ b/scrapy/contrib/throttle.py @@ -1,4 +1,3 @@ -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy import signals from scrapy.utils.httpobj import urlparse_cached @@ -63,8 +62,8 @@ class AutoThrottle(object): if not settings.getbool('AUTOTHROTTLE_ENABLED'): raise NotConfigured self.crawler = crawler - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.response_received, signal=signals.response_received) + crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(self.response_received, signal=signals.response_received) self.START_DELAY = settings.getfloat("AUTOTHROTTLE_START_DELAY", 5.0) self.CONCURRENCY_CHECK_PERIOD = settings.getint("AUTOTHROTTLE_CONCURRENCY_CHECK_PERIOD", 10) self.MAX_CONCURRENCY = settings.getint("AUTOTHROTTLE_MAX_CONCURRENCY", 8) diff --git a/scrapy/contrib/webservice/stats.py b/scrapy/contrib/webservice/stats.py index 623758c20..df17a8a7d 100644 --- a/scrapy/contrib/webservice/stats.py +++ b/scrapy/contrib/webservice/stats.py @@ -1,9 +1,8 @@ from scrapy.webservice import JsonRpcResource -from scrapy.stats import stats class StatsResource(JsonRpcResource): ws_name = 'stats' def __init__(self, crawler): - JsonRpcResource.__init__(self, crawler, stats) + JsonRpcResource.__init__(self, crawler, crawler.stats) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f3d639c18..456b0cce1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -8,7 +8,6 @@ from twisted.internet import reactor, defer from twisted.python.failure import Failure from scrapy.utils.defer import mustbe_deferred -from scrapy.utils.signal import send_catch_log from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache from scrapy.exceptions import ScrapyDeprecationWarning @@ -68,6 +67,7 @@ class Downloader(object): def __init__(self, crawler): self.settings = crawler.settings + self.signals = crawler.signals self.slots = {} self.active = set() self.handlers = DownloadHandlers() @@ -114,7 +114,7 @@ class Downloader(object): def _enqueue_request(self, request, spider, slot): def _downloaded(response): - send_catch_log(signal=signals.response_downloaded, \ + self.signals.send_catch_log(signal=signals.response_downloaded, \ response=response, request=request, spider=spider) return response diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7680df799..f0b4e8696 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -11,13 +11,11 @@ from twisted.internet import defer from twisted.python.failure import Failure from scrapy import log, signals -from scrapy.stats import stats from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request from scrapy.utils.misc import load_object -from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.reactor import CallLaterOnce @@ -53,7 +51,9 @@ class Slot(object): class ExecutionEngine(object): def __init__(self, crawler, spider_closed_callback): + self.crawler = crawler self.settings = crawler.settings + self.signals = crawler.signals self.slots = {} self.running = False self.paused = False @@ -71,7 +71,7 @@ class ExecutionEngine(object): """Start the execution engine""" assert not self.running, "Engine already running" self.start_time = time() - yield send_catch_log_deferred(signal=signals.engine_started) + yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True def stop(self): @@ -195,7 +195,7 @@ class ExecutionEngine(object): response.request = request # tie request to response received log.msg(log.formatter.crawled(request, response, spider), \ level=log.DEBUG, spider=spider) - send_catch_log(signal=signals.response_received, \ + self.signals.send_catch_log(signal=signals.response_received, \ response=response, request=request, spider=spider) return response @@ -218,13 +218,13 @@ class ExecutionEngine(object): spider.name log.msg("Spider opened", spider=spider) nextcall = CallLaterOnce(self._next_request, spider) - scheduler = self.scheduler_cls.from_settings(self.settings) + scheduler = self.scheduler_cls.from_crawler(self.crawler) slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler) self.slots[spider] = slot yield scheduler.open(spider) yield self.scraper.open_spider(spider) - stats.open_spider(spider) - yield send_catch_log_deferred(signals.spider_opened, spider=spider) + self.crawler.stats.open_spider(spider) + yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) slot.nextcall.schedule() def _spider_idle(self, spider): @@ -235,7 +235,7 @@ class ExecutionEngine(object): next loop and this function is guaranteed to be called (at least) once again for this spider. """ - res = send_catch_log(signal=signals.spider_idle, \ + res = self.signals.send_catch_log(signal=signals.spider_idle, \ spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \ for _, x in res): @@ -261,11 +261,11 @@ class ExecutionEngine(object): dfd.addBoth(lambda _: slot.scheduler.close(reason)) dfd.addErrback(log.err, spider=spider) - dfd.addBoth(lambda _: send_catch_log_deferred(signal=signals.spider_closed, \ + dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(signal=signals.spider_closed, \ spider=spider, reason=reason)) dfd.addErrback(log.err, spider=spider) - dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason)) + dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) dfd.addErrback(log.err, spider=spider) dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider)) @@ -284,5 +284,5 @@ class ExecutionEngine(object): @defer.inlineCallbacks def _finish_stopping_engine(self): - yield send_catch_log_deferred(signal=signals.engine_stopped) - yield stats.engine_stopped() + yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + yield self.crawler.stats.engine_stopped() diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index bd97c4f01..f65fe03cb 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,26 +6,27 @@ from scrapy.utils.pqueue import PriorityQueue from scrapy.utils.reqser import request_to_dict, request_from_dict from scrapy.utils.misc import load_object from scrapy.utils.job import job_dir -from scrapy.stats import stats from scrapy import log class Scheduler(object): - def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False): + def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.dqclass = dqclass self.mqclass = mqclass self.logunser = logunser + self.stats = stats @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = dupefilter_cls.from_settings(settings) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS') - return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser) + return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser, crawler.stats) def has_pending_requests(self): return len(self) > 0 @@ -67,11 +68,13 @@ class Scheduler(object): (request, str(e)), level=log.ERROR, spider=self.spider) return else: - stats.inc_value('scheduler/disk_enqueued', spider=self.spider) + if self.stats: + self.stats.inc_value('scheduler/disk_enqueued', spider=self.spider) return True def _mqpush(self, request): - stats.inc_value('scheduler/memory_enqueued', spider=self.spider) + if self.stats: + self.stats.inc_value('scheduler/memory_enqueued', spider=self.spider) self.mqs.push(request, -request.priority) def _dqpop(self): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2e3b2d43e..97bdef81e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -9,14 +9,12 @@ from twisted.internet import defer from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback from scrapy.utils.spider import iterate_spider_output from scrapy.utils.misc import load_object -from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.exceptions import CloseSpider, DropItem from scrapy import signals from scrapy.http import Request, Response from scrapy.item import BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy import log -from scrapy.stats import stats class Slot(object): @@ -68,6 +66,7 @@ class Scraper(object): self.itemproc = itemproc_cls.from_crawler(crawler) self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS') self.crawler = crawler + self.signals = crawler.signals @defer.inlineCallbacks def open_spider(self, spider): @@ -146,9 +145,9 @@ class Scraper(object): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return log.err(_failure, "Spider error processing %s" % request, spider=spider) - send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \ + self.signals.send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \ spider=spider) - stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ + self.crawler.stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ spider=spider) def handle_spider_output(self, result, request, response, spider): @@ -164,7 +163,7 @@ class Scraper(object): from the given spider """ if isinstance(output, Request): - send_catch_log(signal=signals.request_received, request=output, \ + self.signals.send_catch_log(signal=signals.request_received, request=output, \ spider=spider) self.crawler.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): @@ -199,13 +198,13 @@ class Scraper(object): if isinstance(ex, DropItem): log.msg(log.formatter.dropped(item, ex, response, spider), \ level=log.WARNING, spider=spider) - return send_catch_log_deferred(signal=signals.item_dropped, \ + return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \ item=item, spider=spider, exception=output.value) else: log.err(output, 'Error processing %s' % item, spider=spider) else: log.msg(log.formatter.scraped(output, response, spider), \ log.DEBUG, spider=spider) - return send_catch_log_deferred(signal=signals.item_scraped, \ + return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \ item=output, response=response, spider=spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d54bc84c9..77122c743 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -2,10 +2,10 @@ import signal from twisted.internet import reactor, defer -from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.extension import ExtensionManager +from scrapy.signalmanager import SignalManager from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals @@ -16,6 +16,8 @@ class Crawler(object): def __init__(self, settings): self.configured = False self.settings = settings + self.signals = SignalManager(self) + self.stats = load_object(settings['STATS_CLASS'])(self) def install(self): import scrapy.project @@ -65,7 +67,7 @@ class CrawlerProcess(Crawler): def __init__(self, *a, **kw): super(CrawlerProcess, self).__init__(*a, **kw) - dispatcher.connect(self.stop, signals.engine_stopped) + self.signals.connect(self.stop, signals.engine_stopped) install_shutdown_handlers(self._signal_shutdown) def start(self): diff --git a/scrapy/mail.py b/scrapy/mail.py index a7e4c94f7..db5193721 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -17,7 +17,6 @@ from twisted.mail.smtp import ESMTPSenderFactory from scrapy import log from scrapy.exceptions import NotConfigured from scrapy.conf import settings -from scrapy.utils.signal import send_catch_log # signal sent when message is sent @@ -28,13 +27,14 @@ mail_sent = object() class MailSender(object): def __init__(self, smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, \ - smtpport=None, debug=False): + smtpport=None, debug=False, crawler=None): self.smtphost = smtphost or settings['MAIL_HOST'] self.smtpport = smtpport or settings.getint('MAIL_PORT') self.smtpuser = smtpuser or settings['MAIL_USER'] self.smtppass = smtppass or settings['MAIL_PASS'] self.mailfrom = mailfrom or settings['MAIL_FROM'] self.debug = debug + self.signals = crawler.signals if crawler else None if not self.smtphost or not self.mailfrom: raise NotConfigured("MAIL_HOST and MAIL_FROM settings are required") @@ -65,7 +65,8 @@ class MailSender(object): else: msg.set_payload(body) - send_catch_log(signal=mail_sent, to=to, subject=subject, body=body, + if self.signals: + self.signals.send_catch_log(signal=mail_sent, to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg) if self.debug: diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py new file mode 100644 index 000000000..4a3e3d92d --- /dev/null +++ b/scrapy/signalmanager.py @@ -0,0 +1,27 @@ +from scrapy.xlib.pydispatch import dispatcher +from scrapy.utils import signal + +class SignalManager(object): + + def __init__(self, sender=dispatcher.Anonymous): + self.sender = sender + + def connect(self, *a, **kw): + kw.setdefault('sender', self.sender) + return dispatcher.connect(*a, **kw) + + def disconnect(self, *a, **kw): + kw.setdefault('sender', self.sender) + return dispatcher.disconnect(*a, **kw) + + def send_catch_log(self, *a, **kw): + kw.setdefault('sender', self.sender) + return signal.send_catch_log(*a, **kw) + + def send_catch_log_deferred(self, *a, **kw): + kw.setdefault('sender', self.sender) + return signal.send_catch_log_deferred(*a, **kw) + + def disconnect_all(self, *a, **kw): + kw.setdefault('sender', self.sender) + return signal.disconnect_all(*a, **kw) diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index ee89f9e3d..6443c45fa 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -9,7 +9,6 @@ from scrapy import signals from scrapy.interfaces import ISpiderManager from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes -from scrapy.xlib.pydispatch import dispatcher class SpiderManager(object): @@ -22,7 +21,6 @@ class SpiderManager(object): for name in self.spider_modules: for module in walk_modules(name): self._load_spiders(module) - dispatcher.connect(self.close_spider, signals.spider_closed) def _load_spiders(self, module): for spcls in iter_spider_classes(module): @@ -34,7 +32,9 @@ class SpiderManager(object): @classmethod def from_crawler(cls, crawler): - return cls.from_settings(crawler.settings) + sm = cls.from_settings(crawler.settings) + crawler.signals.connect(sm.close_spider, signals.spider_closed) + return sm def create(self, spider_name, **spider_kwargs): try: diff --git a/scrapy/stats.py b/scrapy/stats.py index 37d9c6888..b8128dfc2 100644 --- a/scrapy/stats.py +++ b/scrapy/stats.py @@ -1,9 +1,7 @@ -from scrapy.statscol import DummyStatsCollector -from scrapy.conf import settings -from scrapy.utils.misc import load_object +from scrapy.project import crawler +stats = crawler.stats -# if stats are disabled use a DummyStatsCollector to improve performance -if settings.getbool('STATS_ENABLED'): - stats = load_object(settings['STATS_CLASS'])() -else: - stats = DummyStatsCollector() +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead", + ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/statscol.py b/scrapy/statscol.py index b4ecb2f05..8cbea93cc 100644 --- a/scrapy/statscol.py +++ b/scrapy/statscol.py @@ -3,20 +3,16 @@ Scrapy extension for collecting scraping stats """ import pprint -from scrapy.xlib.pydispatch import dispatcher - from scrapy.signals import stats_spider_opened, stats_spider_closing, \ stats_spider_closed -from scrapy.utils.signal import send_catch_log -from scrapy import signals from scrapy import log -from scrapy.conf import settings class StatsCollector(object): - def __init__(self): - self._dump = settings.getbool('STATS_DUMP') + def __init__(self, crawler): + self._dump = crawler.settings.getbool('STATS_DUMP') self._stats = {None: {}} # None is for global stats + self._signals = crawler.signals def get_value(self, key, default=None, spider=None): return self._stats[spider].get(key, default) @@ -50,12 +46,12 @@ class StatsCollector(object): def open_spider(self, spider): self._stats[spider] = {} - send_catch_log(stats_spider_opened, spider=spider) + self._signals.send_catch_log(stats_spider_opened, spider=spider) def close_spider(self, spider, reason): - send_catch_log(stats_spider_closing, spider=spider, reason=reason) + self._signals.send_catch_log(stats_spider_closing, spider=spider, reason=reason) stats = self._stats.pop(spider) - send_catch_log(stats_spider_closed, spider=spider, reason=reason, \ + self._signals.send_catch_log(stats_spider_closed, spider=spider, reason=reason, \ spider_stats=stats) if self._dump: log.msg("Dumping spider stats:\n" + pprint.pformat(stats), \ @@ -73,8 +69,8 @@ class StatsCollector(object): class MemoryStatsCollector(StatsCollector): - def __init__(self): - super(MemoryStatsCollector, self).__init__() + def __init__(self, crawler): + super(MemoryStatsCollector, self).__init__(crawler) self.spider_stats = {} def _persist_stats(self, stats, spider=None): diff --git a/scrapy/telnet.py b/scrapy/telnet.py index 15eab1a37..474c504af 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -10,11 +10,8 @@ from twisted.conch import manhole, telnet from twisted.conch.insults import insults from twisted.internet import protocol -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured -from scrapy.stats import stats from scrapy import log, signals -from scrapy.utils.signal import send_catch_log from scrapy.utils.trackref import print_live_refs from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp @@ -39,8 +36,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = map(int, crawler.settings.getlist('TELNETCONSOLE_PORT')) self.host = crawler.settings['TELNETCONSOLE_HOST'] - dispatcher.connect(self.start_listening, signals.engine_started) - dispatcher.connect(self.stop_listening, signals.engine_stopped) + self.crawler.signals.connect(self.start_listening, signals.engine_started) + self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod def from_crawler(cls, crawler): @@ -70,7 +67,7 @@ class TelnetConsole(protocol.ServerFactory): 'slot': slot, 'manager': self.crawler, 'extensions': self.crawler.extensions, - 'stats': stats, + 'stats': self.crawler.stats, 'spiders': self.crawler.spiders, 'settings': self.crawler.settings, 'est': lambda: print_engine_status(self.crawler.engine), @@ -80,5 +77,5 @@ class TelnetConsole(protocol.ServerFactory): 'help': "This is Scrapy telnet console. For more info see: " \ "http://doc.scrapy.org/en/latest/topics/telnetconsole.html", } - send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) + self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index ceca8cc80..1474ff9a9 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -3,9 +3,9 @@ import unittest, tempfile, shutil, time from scrapy.http import Response, HtmlResponse, Request from scrapy.spider import BaseSpider from scrapy.contrib.downloadermiddleware.httpcache import FilesystemCacheStorage, HttpCacheMiddleware -from scrapy.stats import stats from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest +from scrapy.utils.test import get_crawler class HttpCacheMiddlewareTest(unittest.TestCase): @@ -13,14 +13,15 @@ class HttpCacheMiddlewareTest(unittest.TestCase): storage_class = FilesystemCacheStorage def setUp(self): + self.crawler = get_crawler() self.spider = BaseSpider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', headers={'User-Agent': 'test'}) self.response = Response('http://www.example.com', headers={'Content-Type': 'text/html'}, body='test body', status=202) - stats.open_spider(self.spider) + self.crawler.stats.open_spider(self.spider) def tearDown(self): - stats.close_spider(self.spider, '') + self.crawler.stats.close_spider(self.spider, '') shutil.rmtree(self.tmpdir) def _get_settings(self, **new_settings): @@ -37,7 +38,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): return self.storage_class(self._get_settings(**new_settings)) def _get_middleware(self, **new_settings): - return HttpCacheMiddleware(self._get_settings(**new_settings)) + return HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats) def test_storage(self): storage = self._get_storage() @@ -58,7 +59,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert storage.retrieve_response(self.spider, self.request) def test_middleware(self): - mw = HttpCacheMiddleware(self._get_settings()) + mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) assert mw.process_request(self.request, self.spider) is None mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) @@ -67,7 +68,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert 'cached' in response.flags def test_different_request_response_urls(self): - mw = HttpCacheMiddleware(self._get_settings()) + mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) req = Request('http://host.com/path') res = Response('http://host2.net/test.html') assert mw.process_request(req, self.spider) is None diff --git a/scrapy/tests/test_downloadermiddleware_stats.py b/scrapy/tests/test_downloadermiddleware_stats.py index 193b8bad1..aab8a2b5e 100644 --- a/scrapy/tests/test_downloadermiddleware_stats.py +++ b/scrapy/tests/test_downloadermiddleware_stats.py @@ -3,35 +3,36 @@ from unittest import TestCase from scrapy.contrib.downloadermiddleware.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spider import BaseSpider -from scrapy.stats import stats +from scrapy.utils.test import get_crawler class TestDownloaderStats(TestCase): def setUp(self): + self.crawler = get_crawler() self.spider = BaseSpider('scrapytest.org') - self.mw = DownloaderStats() + self.mw = DownloaderStats(self.crawler.stats) - stats.open_spider(self.spider) + self.crawler.stats.open_spider(self.spider) self.req = Request('http://scrapytest.org') self.res = Response('scrapytest.org', status=400) def test_process_request(self): self.mw.process_request(self.req, self.spider) - self.assertEqual(stats.get_value('downloader/request_count', \ + self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \ spider=self.spider), 1) def test_process_response(self): self.mw.process_response(self.req, self.res, self.spider) - self.assertEqual(stats.get_value('downloader/response_count', \ + self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \ spider=self.spider), 1) def test_process_exception(self): self.mw.process_exception(self.req, Exception(), self.spider) - self.assertEqual(stats.get_value('downloader/exception_count', \ + self.assertEqual(self.crawler.stats.get_value('downloader/exception_count', \ spider=self.spider), 1) def tearDown(self): - stats.close_spider(self.spider, '') + self.crawler.stats.close_spider(self.spider, '') diff --git a/scrapy/tests/test_mail.py b/scrapy/tests/test_mail.py index 4512997f8..f4e6a10d8 100644 --- a/scrapy/tests/test_mail.py +++ b/scrapy/tests/test_mail.py @@ -1,22 +1,22 @@ from cStringIO import StringIO import unittest -from scrapy.xlib.pydispatch import dispatcher - from scrapy.mail import MailSender, mail_sent +from scrapy.utils.test import get_crawler class MailSenderTest(unittest.TestCase): def setUp(self): self.catched_msg = None - dispatcher.connect(self._catch_mail_sent, signal=mail_sent) + self.crawler = get_crawler() + self.crawler.signals.connect(self._catch_mail_sent, signal=mail_sent) def tearDown(self): - dispatcher.disconnect(self._catch_mail_sent, signal=mail_sent) + self.crawler.signals.disconnect(self._catch_mail_sent, signal=mail_sent) def test_send(self): - mailsender = MailSender(debug=True) + mailsender = MailSender(debug=True, crawler=self.crawler) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body') assert self.catched_msg @@ -36,7 +36,7 @@ class MailSenderTest(unittest.TestCase): attach.seek(0) attachs = [('attachment', 'text/plain', attach)] - mailsender = MailSender(debug=True) + mailsender = MailSender(debug=True, crawler=self.crawler) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', attachs=attachs) diff --git a/scrapy/tests/test_spidermiddleware_depth.py b/scrapy/tests/test_spidermiddleware_depth.py index 6f9cab08a..beace38f9 100644 --- a/scrapy/tests/test_spidermiddleware_depth.py +++ b/scrapy/tests/test_spidermiddleware_depth.py @@ -4,6 +4,7 @@ from scrapy.contrib.spidermiddleware.depth import DepthMiddleware from scrapy.http import Response, Request from scrapy.spider import BaseSpider from scrapy.statscol import StatsCollector +from scrapy.utils.test import get_crawler class TestDepthMiddleware(TestCase): @@ -11,7 +12,7 @@ class TestDepthMiddleware(TestCase): def setUp(self): self.spider = BaseSpider('scrapytest.org') - self.stats = StatsCollector() + self.stats = StatsCollector(get_crawler()) self.stats.open_spider(self.spider) self.mw = DepthMiddleware(1, self.stats, True) diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index a39c0bd0e..f32af4208 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -1,18 +1,19 @@ import unittest from scrapy.spider import BaseSpider -from scrapy.xlib.pydispatch import dispatcher from scrapy.statscol import StatsCollector, DummyStatsCollector from scrapy.signals import stats_spider_opened, stats_spider_closing, \ stats_spider_closed +from scrapy.utils.test import get_crawler class StatsCollectorTest(unittest.TestCase): def setUp(self): + self.crawler = get_crawler() self.spider = BaseSpider('foo') def test_collector(self): - stats = StatsCollector() + stats = StatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) self.assertEqual(stats.get_value('anything'), None) self.assertEqual(stats.get_value('anything', 'default'), 'default') @@ -39,7 +40,7 @@ class StatsCollectorTest(unittest.TestCase): self.assertEqual(stats.get_value('test4'), 7) def test_dummy_collector(self): - stats = DummyStatsCollector() + stats = DummyStatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) self.assertEqual(stats.get_value('anything'), None) self.assertEqual(stats.get_value('anything', 'default'), 'default') @@ -70,11 +71,11 @@ class StatsCollectorTest(unittest.TestCase): assert spider_stats == {'test': 1} signals_catched.add(stats_spider_closed) - dispatcher.connect(spider_opened, signal=stats_spider_opened) - dispatcher.connect(spider_closing, signal=stats_spider_closing) - dispatcher.connect(spider_closed, signal=stats_spider_closed) + self.crawler.signals.connect(spider_opened, signal=stats_spider_opened) + self.crawler.signals.connect(spider_closing, signal=stats_spider_closing) + self.crawler.signals.connect(spider_closed, signal=stats_spider_closed) - stats = StatsCollector() + stats = StatsCollector(self.crawler) stats.open_spider(self.spider) stats.set_value('test', 1, spider=self.spider) self.assertEqual([(self.spider, {'test': 1})], list(stats.iter_spider_stats())) @@ -83,9 +84,9 @@ class StatsCollectorTest(unittest.TestCase): assert stats_spider_closing in signals_catched assert stats_spider_closed in signals_catched - dispatcher.disconnect(spider_opened, signal=stats_spider_opened) - dispatcher.disconnect(spider_closing, signal=stats_spider_closing) - dispatcher.disconnect(spider_closed, signal=stats_spider_closed) + self.crawler.signals.disconnect(spider_opened, signal=stats_spider_opened) + self.crawler.signals.disconnect(spider_closing, signal=stats_spider_closing) + self.crawler.signals.disconnect(spider_closed, signal=stats_spider_closed) if __name__ == "__main__": unittest.main() diff --git a/scrapy/webservice.py b/scrapy/webservice.py index 0cca1b1b0..889dce5ad 100644 --- a/scrapy/webservice.py +++ b/scrapy/webservice.py @@ -6,7 +6,6 @@ See docs/topics/webservice.rst from twisted.web import server, error -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy import log, signals from scrapy.utils.jsonrpc import jsonrpc_server_call @@ -80,8 +79,8 @@ class WebService(server.Site): root.putChild(res.ws_name, res) server.Site.__init__(self, root, logPath=logfile) self.noisy = False - dispatcher.connect(self.start_listening, signals.engine_started) - dispatcher.connect(self.stop_listening, signals.engine_stopped) + crawler.signals.connect(self.start_listening, signals.engine_started) + crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod def from_crawler(cls, crawler):