From 1e12c92b8f4114b75bac7bed714d2e59e12b701d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 6 Mar 2012 08:59:20 -0200 Subject: [PATCH 1/2] Removed signals/stats singletons This change removes singletons for stats collection and signal dispatching facilities, by making them a member of the Crawler class. Here are some examples to illustrates the old and new API: Signals - before: from scrapy import signals from scrapy.xlib.pydispatch import dispatcher dispatcher.connect(self.spider_opened, signals.spider_opened) Signals - now: from scrapy import signals crawler.signals.connect(self.spider.opened, signals.spider_opened) Stats collection - before: from scrapy.stats import stats stats.inc_value('foo') Stats collection - now: crawler.stats.inc_value('foo') Backwards compatibility was retained as much as possible and the old API has been properly flagged with deprecation warnings. --- scrapy/contrib/closespider.py | 9 +++--- scrapy/contrib/corestats.py | 31 ++++++++++--------- .../contrib/downloadermiddleware/httpcache.py | 18 +++++------ .../contrib/downloadermiddleware/robotstxt.py | 6 ++-- scrapy/contrib/downloadermiddleware/stats.py | 22 +++++++------ scrapy/contrib/feedexport.py | 12 ++++--- scrapy/contrib/logstats.py | 21 ++++++++----- scrapy/contrib/memdebug.py | 19 ++++++------ scrapy/contrib/memusage.py | 22 ++++++------- scrapy/contrib/pipeline/images.py | 11 ++----- scrapy/contrib/spidermiddleware/depth.py | 9 ++---- scrapy/contrib/spidermiddleware/offsite.py | 10 ++++-- scrapy/contrib/spiderstate.py | 5 ++- scrapy/contrib/statsmailer.py | 19 ++++++------ scrapy/contrib/throttle.py | 5 ++- scrapy/contrib/webservice/stats.py | 3 +- scrapy/core/downloader/__init__.py | 4 +-- scrapy/core/engine.py | 24 +++++++------- scrapy/core/scheduler.py | 15 +++++---- scrapy/core/scraper.py | 13 ++++---- scrapy/crawler.py | 6 ++-- scrapy/mail.py | 7 +++-- scrapy/signalmanager.py | 27 ++++++++++++++++ scrapy/spidermanager.py | 6 ++-- scrapy/stats.py | 14 ++++----- scrapy/statscol.py | 20 +++++------- scrapy/telnet.py | 11 +++---- .../test_downloadermiddleware_httpcache.py | 13 ++++---- .../tests/test_downloadermiddleware_stats.py | 15 ++++----- scrapy/tests/test_mail.py | 12 +++---- scrapy/tests/test_spidermiddleware_depth.py | 3 +- scrapy/tests/test_stats.py | 21 +++++++------ scrapy/webservice.py | 5 ++- 33 files changed, 232 insertions(+), 206 deletions(-) create mode 100644 scrapy/signalmanager.py diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index f4611f4fc..d9b01e689 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -8,7 +8,6 @@ from collections import defaultdict from twisted.internet import reactor from twisted.python import log as txlog -from scrapy.xlib.pydispatch import dispatcher from scrapy import signals, log @@ -29,12 +28,12 @@ class CloseSpider(object): if self.errorcount: txlog.addObserver(self.catch_log) if self.pagecount: - dispatcher.connect(self.page_count, signal=signals.response_received) + crawler.signals.connect(self.page_count, signal=signals.response_received) if self.timeout: - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) if self.itemcount: - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + crawler.signals.connect(self.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(self.spider_closed, signal=signals.spider_closed) @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py index da25d6a97..f45e78a43 100644 --- a/scrapy/contrib/corestats.py +++ b/scrapy/contrib/corestats.py @@ -3,30 +3,33 @@ Extension for collecting core stats like items scraped and start/finish times """ import datetime -from scrapy.xlib.pydispatch import dispatcher - from scrapy import signals -from scrapy.stats import stats class CoreStats(object): - def __init__(self): - dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened) - dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing) - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.item_dropped, signal=signals.item_dropped) + def __init__(self, stats): + self.stats = stats + + @classmethod + def from_crawler(cls, crawler): + o = cls(crawler.stats) + crawler.signals.connect(o.stats_spider_opened, signal=signals.stats_spider_opened) + crawler.signals.connect(o.stats_spider_closing, signal=signals.stats_spider_closing) + crawler.signals.connect(o.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(o.item_dropped, signal=signals.item_dropped) + return o def stats_spider_opened(self, spider): - stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) + self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) def stats_spider_closing(self, spider, reason): - stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) - stats.set_value('finish_reason', reason, spider=spider) + self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) + self.stats.set_value('finish_reason', reason, spider=spider) def item_scraped(self, item, spider): - stats.inc_value('item_scraped_count', spider=spider) + self.stats.inc_value('item_scraped_count', spider=spider) def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ - stats.inc_value('item_dropped_count', spider=spider) - stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) + self.stats.inc_value('item_dropped_count', spider=spider) + self.stats.inc_value('item_dropped_reasons_count/%s' % reason, spider=spider) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 642f9bda7..cb12ac031 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -5,11 +5,9 @@ import cPickle as pickle from w3lib.http import headers_dict_to_raw, headers_raw_to_dict -from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.http import Headers from scrapy.exceptions import NotConfigured, IgnoreRequest -from scrapy.stats import stats from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.httpobj import urlparse_cached @@ -19,19 +17,21 @@ from scrapy.utils.project import data_path class HttpCacheMiddleware(object): - def __init__(self, settings): + def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + self.stats = stats @classmethod def from_crawler(cls, crawler): - return cls(crawler.settings) + o = cls(crawler.settings, crawler.stats) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o def spider_opened(self, spider): self.storage.open_spider(spider) @@ -45,10 +45,10 @@ class HttpCacheMiddleware(object): response = self.storage.retrieve_response(spider, request) if response and self.is_cacheable_response(response): response.flags.append('cached') - stats.inc_value('httpcache/hit', spider=spider) + self.stats.inc_value('httpcache/hit', spider=spider) return response - stats.inc_value('httpcache/miss', spider=spider) + self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) @@ -57,7 +57,7 @@ class HttpCacheMiddleware(object): and self.is_cacheable_response(response) and 'cached' not in response.flags): self.storage.store_response(spider, request, response) - stats.inc_value('httpcache/store', spider=spider) + self.stats.inc_value('httpcache/store', spider=spider) return response def is_cacheable_response(self, response): diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index bf64677f8..8e9feda81 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -6,8 +6,6 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. import robotparser -from scrapy.xlib.pydispatch import dispatcher - from scrapy import signals, log from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request @@ -24,8 +22,8 @@ class RobotsTxtMiddleware(object): self._parsers = {} self._spider_netlocs = {} self._useragents = {} - dispatcher.connect(self.spider_opened, signals.spider_opened) - dispatcher.connect(self.spider_closed, signals.spider_closed) + crawler.signals.connect(self.spider_opened, signals.spider_opened) + crawler.signals.connect(self.spider_closed, signals.spider_closed) @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/downloadermiddleware/stats.py b/scrapy/contrib/downloadermiddleware/stats.py index 32bad0bb3..9c0ad90a5 100644 --- a/scrapy/contrib/downloadermiddleware/stats.py +++ b/scrapy/contrib/downloadermiddleware/stats.py @@ -1,30 +1,32 @@ from scrapy.exceptions import NotConfigured from scrapy.utils.request import request_httprepr from scrapy.utils.response import response_httprepr -from scrapy.stats import stats class DownloaderStats(object): + def __init__(self, stats): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('DOWNLOADER_STATS'): raise NotConfigured - return cls() + return cls(crawler.stats) def process_request(self, request, spider): - stats.inc_value('downloader/request_count', spider=spider) - stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) + self.stats.inc_value('downloader/request_count', spider=spider) + self.stats.inc_value('downloader/request_method_count/%s' % request.method, spider=spider) reqlen = len(request_httprepr(request)) - stats.inc_value('downloader/request_bytes', reqlen, spider=spider) + self.stats.inc_value('downloader/request_bytes', reqlen, spider=spider) def process_response(self, request, response, spider): - stats.inc_value('downloader/response_count', spider=spider) - stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) + self.stats.inc_value('downloader/response_count', spider=spider) + self.stats.inc_value('downloader/response_status_count/%s' % response.status, spider=spider) reslen = len(response_httprepr(response)) - stats.inc_value('downloader/response_bytes', reslen, spider=spider) + self.stats.inc_value('downloader/response_bytes', reslen, spider=spider) return response def process_exception(self, request, exception, spider): ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__) - stats.inc_value('downloader/exception_count', spider=spider) - stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) + self.stats.inc_value('downloader/exception_count', spider=spider) + self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider) diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index cdda804f7..fd84c59a1 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -15,7 +15,6 @@ from twisted.internet import defer, threads from w3lib.url import file_uri_to_path from scrapy import log, signals -from scrapy.xlib.pydispatch import dispatcher from scrapy.utils.ftp import ftp_makedirs_cwd from scrapy.exceptions import NotConfigured from scrapy.utils.misc import load_object @@ -149,9 +148,14 @@ class FeedExporter(object): uripar = settings['FEED_URI_PARAMS'] self._uripar = load_object(uripar) if uripar else lambda x, y: None self.slots = {} - dispatcher.connect(self.open_spider, signals.spider_opened) - dispatcher.connect(self.close_spider, signals.spider_closed) - dispatcher.connect(self.item_scraped, signals.item_scraped) + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.open_spider, signals.spider_opened) + crawler.signals.connect(o.close_spider, signals.spider_closed) + crawler.signals.connect(o.item_scraped, signals.item_scraped) + return o @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/logstats.py b/scrapy/contrib/logstats.py index a22c1cb2a..47b762b07 100644 --- a/scrapy/contrib/logstats.py +++ b/scrapy/contrib/logstats.py @@ -1,6 +1,5 @@ from twisted.internet import task -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy import log, signals @@ -19,12 +18,20 @@ class LogStats(object): self.interval = interval self.slots = {} self.multiplier = 60.0 / self.interval - dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.response_received, signal=signals.response_received) - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - dispatcher.connect(self.engine_started, signal=signals.engine_started) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) + + @classmethod + def from_crawler(cls, crawler): + interval = settings.getfloat('LOGSTATS_INTERVAL') + if not interval: + raise NotConfigured + o = cls(interval) + crawler.signals.connect(o.item_scraped, signal=signals.item_scraped) + crawler.signals.connect(o.response_received, signal=signals.response_received) + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + crawler.signals.connect(o.engine_started, signal=signals.engine_started) + crawler.signals.connect(o.engine_stopped, signal=signals.engine_stopped) + return o @classmethod def from_crawler(cls, crawler): diff --git a/scrapy/contrib/memdebug.py b/scrapy/contrib/memdebug.py index cb9051fd2..8965a1ebd 100644 --- a/scrapy/contrib/memdebug.py +++ b/scrapy/contrib/memdebug.py @@ -6,30 +6,29 @@ See documentation in docs/topics/extensions.rst import gc -from scrapy.xlib.pydispatch import dispatcher - from scrapy import signals from scrapy.exceptions import NotConfigured -from scrapy.stats import stats from scrapy.utils.trackref import live_refs class MemoryDebugger(object): - def __init__(self, trackrefs=False): + def __init__(self, stats, trackrefs=False): try: import libxml2 self.libxml2 = libxml2 except ImportError: self.libxml2 = None + self.stats = stats self.trackrefs = trackrefs - dispatcher.connect(self.engine_started, signals.engine_started) - dispatcher.connect(self.engine_stopped, signals.engine_stopped) @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('MEMDEBUG_ENABLED'): raise NotConfigured - return cls(crawler.settings.getbool('TRACK_REFS')) + o = cls(crawler.stats, crawler.settings.getbool('TRACK_REFS')) + crawler.signals.connect(o.engine_started, signals.engine_started) + crawler.signals.connect(o.engine_stopped, signals.engine_stopped) + return o def engine_started(self): if self.libxml2: @@ -38,11 +37,11 @@ class MemoryDebugger(object): def engine_stopped(self): if self.libxml2: self.libxml2.cleanupParser() - stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) + self.stats.set_value('memdebug/libxml2_leaked_bytes', self.libxml2.debugMemory(1)) gc.collect() - stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) + self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage)) if self.trackrefs: for cls, wdict in live_refs.iteritems(): if not wdict: continue - stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict)) + self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict)) diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 23ef6ec8e..6388c42af 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -9,12 +9,9 @@ from pprint import pformat from twisted.internet import task -from scrapy.xlib.pydispatch import dispatcher -from scrapy import signals -from scrapy import log +from scrapy import signals, log from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.stats import stats from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported from scrapy.utils.engine import get_engine_status @@ -33,8 +30,8 @@ class MemoryUsage(object): self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 self.report = crawler.settings.getbool('MEMUSAGE_REPORT') self.mail = MailSender() - dispatcher.connect(self.engine_started, signal=signals.engine_started) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) + crawler.signals.connect(self.engine_started, signal=signals.engine_started) + crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped) @classmethod def from_crawler(cls, crawler): @@ -44,7 +41,7 @@ class MemoryUsage(object): return get_vmvalue_from_procfs('VmRSS') def engine_started(self): - stats.set_value('memusage/startup', self.get_virtual_size()) + self.crawler.stats.set_value('memusage/startup', self.get_virtual_size()) self.tasks = [] tsk = task.LoopingCall(self.update) self.tasks.append(tsk) @@ -64,18 +61,18 @@ class MemoryUsage(object): tsk.stop() def update(self): - stats.max_value('memusage/max', self.get_virtual_size()) + self.crawler.stats.max_value('memusage/max', self.get_virtual_size()) def _check_limit(self): if self.get_virtual_size() > self.limit: - stats.set_value('memusage/limit_reached', 1) + self.crawler.stats.set_value('memusage/limit_reached', 1) mem = self.limit/1024/1024 log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR) if self.notify_mails: subj = "%s terminated: memory usage exceeded %dM at %s" % \ (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) - stats.set_value('memusage/limit_notified', 1) + self.crawler.stats.set_value('memusage/limit_notified', 1) open_spiders = self.crawler.engine.open_spiders if open_spiders: for spider in open_spiders: @@ -87,18 +84,19 @@ class MemoryUsage(object): if self.warned: # warn only once return if self.get_virtual_size() > self.warning: - stats.set_value('memusage/warning_reached', 1) + self.crawler.stats.set_value('memusage/warning_reached', 1) mem = self.warning/1024/1024 log.msg("Memory usage reached %dM" % mem, level=log.WARNING) if self.notify_mails: subj = "%s warning: memory usage reached %dM at %s" % \ (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) - stats.set_value('memusage/warning_notified', 1) + self.crawler.stats.set_value('memusage/warning_notified', 1) self.warned = True def _send_report(self, rcpts, subject): """send notification mail with some additional useful info""" + stats = self.crawler.stats s = "Memory usage at engine startup : %dM\r\n" % (stats.get_value('memusage/startup')/1024/1024) s += "Maximum memory usage : %dM\r\n" % (stats.get_value('memusage/max')/1024/1024) s += "Current memory usage : %dM\r\n" % (self.get_virtual_size()/1024/1024) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 29a5910e7..90ab80c7e 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -15,12 +15,9 @@ from collections import defaultdict from twisted.internet import defer, threads from PIL import Image -from scrapy.xlib.pydispatch import dispatcher from scrapy import log -from scrapy.stats import stats from scrapy.utils.misc import md5sum from scrapy.http import Request -from scrapy import signals from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest from scrapy.contrib.pipeline.media import MediaPipeline @@ -40,10 +37,6 @@ class FSImagesStore(object): self.basedir = basedir self._mkdir(self.basedir) self.created_directories = defaultdict(set) - dispatcher.connect(self.spider_closed, signals.spider_closed) - - def spider_closed(self, spider): - self.created_directories.pop(spider.name, None) def persist_image(self, key, image, buf, info): absolute_path = self._get_filesystem_path(key) @@ -274,8 +267,8 @@ class ImagesPipeline(MediaPipeline): yield thumb_key, thumb_image, thumb_buf def inc_stats(self, spider, status): - stats.inc_value('image_count', spider=spider) - stats.inc_value('image_status_count/%s' % status, spider=spider) + spider.crawler.stats.inc_value('image_count', spider=spider) + spider.crawler.stats.inc_value('image_status_count/%s' % status, spider=spider) def convert_image(self, image, size=None): if image.format == 'PNG' and image.mode == 'RGBA': diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index 4023b4009..438051cb5 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -19,16 +19,13 @@ class DepthMiddleware(object): self.prio = prio @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings maxdepth = settings.getint('DEPTH_LIMIT') usestats = settings.getbool('DEPTH_STATS') verbose = settings.getbool('DEPTH_STATS_VERBOSE') prio = settings.getint('DEPTH_PRIORITY') - if usestats: - from scrapy.stats import stats - else: - stats = None - return cls(maxdepth, stats, verbose, prio) + return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): def _filter(request): diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index acb616043..2010ca571 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -6,7 +6,6 @@ See documentation in docs/topics/spider-middleware.rst import re -from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -17,8 +16,13 @@ class OffsiteMiddleware(object): def __init__(self): self.host_regexes = {} self.domains_seen = {} - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) + return o def process_spider_output(self, response, result, spider): for x in result: diff --git a/scrapy/contrib/spiderstate.py b/scrapy/contrib/spiderstate.py index 0347ccbbb..edf0c2946 100644 --- a/scrapy/contrib/spiderstate.py +++ b/scrapy/contrib/spiderstate.py @@ -1,7 +1,6 @@ import os, cPickle as pickle from scrapy import signals -from scrapy.xlib.pydispatch import dispatcher class SpiderState(object): """Store and load spider state during a scraping job""" @@ -12,8 +11,8 @@ class SpiderState(object): @classmethod def from_crawler(cls, crawler): obj = cls(crawler.settings.get('JOBDIR')) - dispatcher.connect(obj.spider_closed, signal=signals.spider_closed) - dispatcher.connect(obj.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(obj.spider_closed, signal=signals.spider_closed) + crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened) return obj def spider_closed(self, spider): diff --git a/scrapy/contrib/statsmailer.py b/scrapy/contrib/statsmailer.py index 9dc5161ec..17ad2f377 100644 --- a/scrapy/contrib/statsmailer.py +++ b/scrapy/contrib/statsmailer.py @@ -4,30 +4,29 @@ StatsMailer extension sends an email when a spider finishes scraping. Use STATSMAILER_RCPTS setting to enable and give the recipient mail address """ -from scrapy.xlib.pydispatch import dispatcher - -from scrapy.stats import stats from scrapy import signals from scrapy.mail import MailSender from scrapy.exceptions import NotConfigured class StatsMailer(object): - def __init__(self, recipients): + def __init__(self, stats, recipients): + self.stats = stats self.recipients = recipients - if not self.recipients: - raise NotConfigured - dispatcher.connect(self.stats_spider_closed, signal=signals.stats_spider_closed) @classmethod def from_crawler(cls, crawler): recipients = crawler.settings.getlist("STATSMAILER_RCPTS") - return cls(recipients) - + if not recipients: + raise NotConfigured + o = cls(crawler.stats, recipients) + crawler.connect(o.stats_spider_closed, signal=signals.stats_spider_closed) + return o + def stats_spider_closed(self, spider, spider_stats): mail = MailSender() body = "Global stats\n\n" - body += "\n".join("%-50s : %s" % i for i in stats.get_stats().items()) + body += "\n".join("%-50s : %s" % i for i in self.stats.get_stats().items()) body += "\n\n%s stats\n\n" % spider.name body += "\n".join("%-50s : %s" % i for i in spider_stats.items()) mail.send(self.recipients, "Scrapy stats for: %s" % spider.name, body) diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py index ba462a55d..22a47aa2a 100644 --- a/scrapy/contrib/throttle.py +++ b/scrapy/contrib/throttle.py @@ -1,4 +1,3 @@ -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy import signals from scrapy.utils.httpobj import urlparse_cached @@ -63,8 +62,8 @@ class AutoThrottle(object): if not settings.getbool('AUTOTHROTTLE_ENABLED'): raise NotConfigured self.crawler = crawler - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.response_received, signal=signals.response_received) + crawler.signals.connect(self.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(self.response_received, signal=signals.response_received) self.START_DELAY = settings.getfloat("AUTOTHROTTLE_START_DELAY", 5.0) self.CONCURRENCY_CHECK_PERIOD = settings.getint("AUTOTHROTTLE_CONCURRENCY_CHECK_PERIOD", 10) self.MAX_CONCURRENCY = settings.getint("AUTOTHROTTLE_MAX_CONCURRENCY", 8) diff --git a/scrapy/contrib/webservice/stats.py b/scrapy/contrib/webservice/stats.py index 623758c20..df17a8a7d 100644 --- a/scrapy/contrib/webservice/stats.py +++ b/scrapy/contrib/webservice/stats.py @@ -1,9 +1,8 @@ from scrapy.webservice import JsonRpcResource -from scrapy.stats import stats class StatsResource(JsonRpcResource): ws_name = 'stats' def __init__(self, crawler): - JsonRpcResource.__init__(self, crawler, stats) + JsonRpcResource.__init__(self, crawler, crawler.stats) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f3d639c18..456b0cce1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -8,7 +8,6 @@ from twisted.internet import reactor, defer from twisted.python.failure import Failure from scrapy.utils.defer import mustbe_deferred -from scrapy.utils.signal import send_catch_log from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache from scrapy.exceptions import ScrapyDeprecationWarning @@ -68,6 +67,7 @@ class Downloader(object): def __init__(self, crawler): self.settings = crawler.settings + self.signals = crawler.signals self.slots = {} self.active = set() self.handlers = DownloadHandlers() @@ -114,7 +114,7 @@ class Downloader(object): def _enqueue_request(self, request, spider, slot): def _downloaded(response): - send_catch_log(signal=signals.response_downloaded, \ + self.signals.send_catch_log(signal=signals.response_downloaded, \ response=response, request=request, spider=spider) return response diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7680df799..f0b4e8696 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -11,13 +11,11 @@ from twisted.internet import defer from twisted.python.failure import Failure from scrapy import log, signals -from scrapy.stats import stats from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request from scrapy.utils.misc import load_object -from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.utils.reactor import CallLaterOnce @@ -53,7 +51,9 @@ class Slot(object): class ExecutionEngine(object): def __init__(self, crawler, spider_closed_callback): + self.crawler = crawler self.settings = crawler.settings + self.signals = crawler.signals self.slots = {} self.running = False self.paused = False @@ -71,7 +71,7 @@ class ExecutionEngine(object): """Start the execution engine""" assert not self.running, "Engine already running" self.start_time = time() - yield send_catch_log_deferred(signal=signals.engine_started) + yield self.signals.send_catch_log_deferred(signal=signals.engine_started) self.running = True def stop(self): @@ -195,7 +195,7 @@ class ExecutionEngine(object): response.request = request # tie request to response received log.msg(log.formatter.crawled(request, response, spider), \ level=log.DEBUG, spider=spider) - send_catch_log(signal=signals.response_received, \ + self.signals.send_catch_log(signal=signals.response_received, \ response=response, request=request, spider=spider) return response @@ -218,13 +218,13 @@ class ExecutionEngine(object): spider.name log.msg("Spider opened", spider=spider) nextcall = CallLaterOnce(self._next_request, spider) - scheduler = self.scheduler_cls.from_settings(self.settings) + scheduler = self.scheduler_cls.from_crawler(self.crawler) slot = Slot(start_requests or (), close_if_idle, nextcall, scheduler) self.slots[spider] = slot yield scheduler.open(spider) yield self.scraper.open_spider(spider) - stats.open_spider(spider) - yield send_catch_log_deferred(signals.spider_opened, spider=spider) + self.crawler.stats.open_spider(spider) + yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider) slot.nextcall.schedule() def _spider_idle(self, spider): @@ -235,7 +235,7 @@ class ExecutionEngine(object): next loop and this function is guaranteed to be called (at least) once again for this spider. """ - res = send_catch_log(signal=signals.spider_idle, \ + res = self.signals.send_catch_log(signal=signals.spider_idle, \ spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \ for _, x in res): @@ -261,11 +261,11 @@ class ExecutionEngine(object): dfd.addBoth(lambda _: slot.scheduler.close(reason)) dfd.addErrback(log.err, spider=spider) - dfd.addBoth(lambda _: send_catch_log_deferred(signal=signals.spider_closed, \ + dfd.addBoth(lambda _: self.signals.send_catch_log_deferred(signal=signals.spider_closed, \ spider=spider, reason=reason)) dfd.addErrback(log.err, spider=spider) - dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason)) + dfd.addBoth(lambda _: self.crawler.stats.close_spider(spider, reason=reason)) dfd.addErrback(log.err, spider=spider) dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider)) @@ -284,5 +284,5 @@ class ExecutionEngine(object): @defer.inlineCallbacks def _finish_stopping_engine(self): - yield send_catch_log_deferred(signal=signals.engine_stopped) - yield stats.engine_stopped() + yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped) + yield self.crawler.stats.engine_stopped() diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index bd97c4f01..f65fe03cb 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -6,26 +6,27 @@ from scrapy.utils.pqueue import PriorityQueue from scrapy.utils.reqser import request_to_dict, request_from_dict from scrapy.utils.misc import load_object from scrapy.utils.job import job_dir -from scrapy.stats import stats from scrapy import log class Scheduler(object): - def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False): + def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.dqclass = dqclass self.mqclass = mqclass self.logunser = logunser + self.stats = stats @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = dupefilter_cls.from_settings(settings) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS') - return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser) + return cls(dupefilter, job_dir(settings), dqclass, mqclass, logunser, crawler.stats) def has_pending_requests(self): return len(self) > 0 @@ -67,11 +68,13 @@ class Scheduler(object): (request, str(e)), level=log.ERROR, spider=self.spider) return else: - stats.inc_value('scheduler/disk_enqueued', spider=self.spider) + if self.stats: + self.stats.inc_value('scheduler/disk_enqueued', spider=self.spider) return True def _mqpush(self, request): - stats.inc_value('scheduler/memory_enqueued', spider=self.spider) + if self.stats: + self.stats.inc_value('scheduler/memory_enqueued', spider=self.spider) self.mqs.push(request, -request.priority) def _dqpop(self): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2e3b2d43e..97bdef81e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -9,14 +9,12 @@ from twisted.internet import defer from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback from scrapy.utils.spider import iterate_spider_output from scrapy.utils.misc import load_object -from scrapy.utils.signal import send_catch_log, send_catch_log_deferred from scrapy.exceptions import CloseSpider, DropItem from scrapy import signals from scrapy.http import Request, Response from scrapy.item import BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy import log -from scrapy.stats import stats class Slot(object): @@ -68,6 +66,7 @@ class Scraper(object): self.itemproc = itemproc_cls.from_crawler(crawler) self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS') self.crawler = crawler + self.signals = crawler.signals @defer.inlineCallbacks def open_spider(self, spider): @@ -146,9 +145,9 @@ class Scraper(object): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return log.err(_failure, "Spider error processing %s" % request, spider=spider) - send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \ + self.signals.send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \ spider=spider) - stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ + self.crawler.stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ spider=spider) def handle_spider_output(self, result, request, response, spider): @@ -164,7 +163,7 @@ class Scraper(object): from the given spider """ if isinstance(output, Request): - send_catch_log(signal=signals.request_received, request=output, \ + self.signals.send_catch_log(signal=signals.request_received, request=output, \ spider=spider) self.crawler.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): @@ -199,13 +198,13 @@ class Scraper(object): if isinstance(ex, DropItem): log.msg(log.formatter.dropped(item, ex, response, spider), \ level=log.WARNING, spider=spider) - return send_catch_log_deferred(signal=signals.item_dropped, \ + return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \ item=item, spider=spider, exception=output.value) else: log.err(output, 'Error processing %s' % item, spider=spider) else: log.msg(log.formatter.scraped(output, response, spider), \ log.DEBUG, spider=spider) - return send_catch_log_deferred(signal=signals.item_scraped, \ + return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \ item=output, response=response, spider=spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d54bc84c9..77122c743 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -2,10 +2,10 @@ import signal from twisted.internet import reactor, defer -from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import ExecutionEngine from scrapy.resolver import CachingThreadedResolver from scrapy.extension import ExtensionManager +from scrapy.signalmanager import SignalManager from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals @@ -16,6 +16,8 @@ class Crawler(object): def __init__(self, settings): self.configured = False self.settings = settings + self.signals = SignalManager(self) + self.stats = load_object(settings['STATS_CLASS'])(self) def install(self): import scrapy.project @@ -65,7 +67,7 @@ class CrawlerProcess(Crawler): def __init__(self, *a, **kw): super(CrawlerProcess, self).__init__(*a, **kw) - dispatcher.connect(self.stop, signals.engine_stopped) + self.signals.connect(self.stop, signals.engine_stopped) install_shutdown_handlers(self._signal_shutdown) def start(self): diff --git a/scrapy/mail.py b/scrapy/mail.py index a7e4c94f7..db5193721 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -17,7 +17,6 @@ from twisted.mail.smtp import ESMTPSenderFactory from scrapy import log from scrapy.exceptions import NotConfigured from scrapy.conf import settings -from scrapy.utils.signal import send_catch_log # signal sent when message is sent @@ -28,13 +27,14 @@ mail_sent = object() class MailSender(object): def __init__(self, smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, \ - smtpport=None, debug=False): + smtpport=None, debug=False, crawler=None): self.smtphost = smtphost or settings['MAIL_HOST'] self.smtpport = smtpport or settings.getint('MAIL_PORT') self.smtpuser = smtpuser or settings['MAIL_USER'] self.smtppass = smtppass or settings['MAIL_PASS'] self.mailfrom = mailfrom or settings['MAIL_FROM'] self.debug = debug + self.signals = crawler.signals if crawler else None if not self.smtphost or not self.mailfrom: raise NotConfigured("MAIL_HOST and MAIL_FROM settings are required") @@ -65,7 +65,8 @@ class MailSender(object): else: msg.set_payload(body) - send_catch_log(signal=mail_sent, to=to, subject=subject, body=body, + if self.signals: + self.signals.send_catch_log(signal=mail_sent, to=to, subject=subject, body=body, cc=cc, attach=attachs, msg=msg) if self.debug: diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py new file mode 100644 index 000000000..4a3e3d92d --- /dev/null +++ b/scrapy/signalmanager.py @@ -0,0 +1,27 @@ +from scrapy.xlib.pydispatch import dispatcher +from scrapy.utils import signal + +class SignalManager(object): + + def __init__(self, sender=dispatcher.Anonymous): + self.sender = sender + + def connect(self, *a, **kw): + kw.setdefault('sender', self.sender) + return dispatcher.connect(*a, **kw) + + def disconnect(self, *a, **kw): + kw.setdefault('sender', self.sender) + return dispatcher.disconnect(*a, **kw) + + def send_catch_log(self, *a, **kw): + kw.setdefault('sender', self.sender) + return signal.send_catch_log(*a, **kw) + + def send_catch_log_deferred(self, *a, **kw): + kw.setdefault('sender', self.sender) + return signal.send_catch_log_deferred(*a, **kw) + + def disconnect_all(self, *a, **kw): + kw.setdefault('sender', self.sender) + return signal.disconnect_all(*a, **kw) diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index ee89f9e3d..6443c45fa 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -9,7 +9,6 @@ from scrapy import signals from scrapy.interfaces import ISpiderManager from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes -from scrapy.xlib.pydispatch import dispatcher class SpiderManager(object): @@ -22,7 +21,6 @@ class SpiderManager(object): for name in self.spider_modules: for module in walk_modules(name): self._load_spiders(module) - dispatcher.connect(self.close_spider, signals.spider_closed) def _load_spiders(self, module): for spcls in iter_spider_classes(module): @@ -34,7 +32,9 @@ class SpiderManager(object): @classmethod def from_crawler(cls, crawler): - return cls.from_settings(crawler.settings) + sm = cls.from_settings(crawler.settings) + crawler.signals.connect(sm.close_spider, signals.spider_closed) + return sm def create(self, spider_name, **spider_kwargs): try: diff --git a/scrapy/stats.py b/scrapy/stats.py index 37d9c6888..b8128dfc2 100644 --- a/scrapy/stats.py +++ b/scrapy/stats.py @@ -1,9 +1,7 @@ -from scrapy.statscol import DummyStatsCollector -from scrapy.conf import settings -from scrapy.utils.misc import load_object +from scrapy.project import crawler +stats = crawler.stats -# if stats are disabled use a DummyStatsCollector to improve performance -if settings.getbool('STATS_ENABLED'): - stats = load_object(settings['STATS_CLASS'])() -else: - stats = DummyStatsCollector() +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.stats` is deprecated, use `crawler.stats` attribute instead", + ScrapyDeprecationWarning, stacklevel=2) diff --git a/scrapy/statscol.py b/scrapy/statscol.py index b4ecb2f05..8cbea93cc 100644 --- a/scrapy/statscol.py +++ b/scrapy/statscol.py @@ -3,20 +3,16 @@ Scrapy extension for collecting scraping stats """ import pprint -from scrapy.xlib.pydispatch import dispatcher - from scrapy.signals import stats_spider_opened, stats_spider_closing, \ stats_spider_closed -from scrapy.utils.signal import send_catch_log -from scrapy import signals from scrapy import log -from scrapy.conf import settings class StatsCollector(object): - def __init__(self): - self._dump = settings.getbool('STATS_DUMP') + def __init__(self, crawler): + self._dump = crawler.settings.getbool('STATS_DUMP') self._stats = {None: {}} # None is for global stats + self._signals = crawler.signals def get_value(self, key, default=None, spider=None): return self._stats[spider].get(key, default) @@ -50,12 +46,12 @@ class StatsCollector(object): def open_spider(self, spider): self._stats[spider] = {} - send_catch_log(stats_spider_opened, spider=spider) + self._signals.send_catch_log(stats_spider_opened, spider=spider) def close_spider(self, spider, reason): - send_catch_log(stats_spider_closing, spider=spider, reason=reason) + self._signals.send_catch_log(stats_spider_closing, spider=spider, reason=reason) stats = self._stats.pop(spider) - send_catch_log(stats_spider_closed, spider=spider, reason=reason, \ + self._signals.send_catch_log(stats_spider_closed, spider=spider, reason=reason, \ spider_stats=stats) if self._dump: log.msg("Dumping spider stats:\n" + pprint.pformat(stats), \ @@ -73,8 +69,8 @@ class StatsCollector(object): class MemoryStatsCollector(StatsCollector): - def __init__(self): - super(MemoryStatsCollector, self).__init__() + def __init__(self, crawler): + super(MemoryStatsCollector, self).__init__(crawler) self.spider_stats = {} def _persist_stats(self, stats, spider=None): diff --git a/scrapy/telnet.py b/scrapy/telnet.py index 15eab1a37..474c504af 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -10,11 +10,8 @@ from twisted.conch import manhole, telnet from twisted.conch.insults import insults from twisted.internet import protocol -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured -from scrapy.stats import stats from scrapy import log, signals -from scrapy.utils.signal import send_catch_log from scrapy.utils.trackref import print_live_refs from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp @@ -39,8 +36,8 @@ class TelnetConsole(protocol.ServerFactory): self.noisy = False self.portrange = map(int, crawler.settings.getlist('TELNETCONSOLE_PORT')) self.host = crawler.settings['TELNETCONSOLE_HOST'] - dispatcher.connect(self.start_listening, signals.engine_started) - dispatcher.connect(self.stop_listening, signals.engine_stopped) + self.crawler.signals.connect(self.start_listening, signals.engine_started) + self.crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod def from_crawler(cls, crawler): @@ -70,7 +67,7 @@ class TelnetConsole(protocol.ServerFactory): 'slot': slot, 'manager': self.crawler, 'extensions': self.crawler.extensions, - 'stats': stats, + 'stats': self.crawler.stats, 'spiders': self.crawler.spiders, 'settings': self.crawler.settings, 'est': lambda: print_engine_status(self.crawler.engine), @@ -80,5 +77,5 @@ class TelnetConsole(protocol.ServerFactory): 'help': "This is Scrapy telnet console. For more info see: " \ "http://doc.scrapy.org/en/latest/topics/telnetconsole.html", } - send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) + self.crawler.signals.send_catch_log(update_telnet_vars, telnet_vars=telnet_vars) return telnet_vars diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index ceca8cc80..1474ff9a9 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -3,9 +3,9 @@ import unittest, tempfile, shutil, time from scrapy.http import Response, HtmlResponse, Request from scrapy.spider import BaseSpider from scrapy.contrib.downloadermiddleware.httpcache import FilesystemCacheStorage, HttpCacheMiddleware -from scrapy.stats import stats from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest +from scrapy.utils.test import get_crawler class HttpCacheMiddlewareTest(unittest.TestCase): @@ -13,14 +13,15 @@ class HttpCacheMiddlewareTest(unittest.TestCase): storage_class = FilesystemCacheStorage def setUp(self): + self.crawler = get_crawler() self.spider = BaseSpider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', headers={'User-Agent': 'test'}) self.response = Response('http://www.example.com', headers={'Content-Type': 'text/html'}, body='test body', status=202) - stats.open_spider(self.spider) + self.crawler.stats.open_spider(self.spider) def tearDown(self): - stats.close_spider(self.spider, '') + self.crawler.stats.close_spider(self.spider, '') shutil.rmtree(self.tmpdir) def _get_settings(self, **new_settings): @@ -37,7 +38,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): return self.storage_class(self._get_settings(**new_settings)) def _get_middleware(self, **new_settings): - return HttpCacheMiddleware(self._get_settings(**new_settings)) + return HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats) def test_storage(self): storage = self._get_storage() @@ -58,7 +59,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert storage.retrieve_response(self.spider, self.request) def test_middleware(self): - mw = HttpCacheMiddleware(self._get_settings()) + mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) assert mw.process_request(self.request, self.spider) is None mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) @@ -67,7 +68,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert 'cached' in response.flags def test_different_request_response_urls(self): - mw = HttpCacheMiddleware(self._get_settings()) + mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) req = Request('http://host.com/path') res = Response('http://host2.net/test.html') assert mw.process_request(req, self.spider) is None diff --git a/scrapy/tests/test_downloadermiddleware_stats.py b/scrapy/tests/test_downloadermiddleware_stats.py index 193b8bad1..aab8a2b5e 100644 --- a/scrapy/tests/test_downloadermiddleware_stats.py +++ b/scrapy/tests/test_downloadermiddleware_stats.py @@ -3,35 +3,36 @@ from unittest import TestCase from scrapy.contrib.downloadermiddleware.stats import DownloaderStats from scrapy.http import Request, Response from scrapy.spider import BaseSpider -from scrapy.stats import stats +from scrapy.utils.test import get_crawler class TestDownloaderStats(TestCase): def setUp(self): + self.crawler = get_crawler() self.spider = BaseSpider('scrapytest.org') - self.mw = DownloaderStats() + self.mw = DownloaderStats(self.crawler.stats) - stats.open_spider(self.spider) + self.crawler.stats.open_spider(self.spider) self.req = Request('http://scrapytest.org') self.res = Response('scrapytest.org', status=400) def test_process_request(self): self.mw.process_request(self.req, self.spider) - self.assertEqual(stats.get_value('downloader/request_count', \ + self.assertEqual(self.crawler.stats.get_value('downloader/request_count', \ spider=self.spider), 1) def test_process_response(self): self.mw.process_response(self.req, self.res, self.spider) - self.assertEqual(stats.get_value('downloader/response_count', \ + self.assertEqual(self.crawler.stats.get_value('downloader/response_count', \ spider=self.spider), 1) def test_process_exception(self): self.mw.process_exception(self.req, Exception(), self.spider) - self.assertEqual(stats.get_value('downloader/exception_count', \ + self.assertEqual(self.crawler.stats.get_value('downloader/exception_count', \ spider=self.spider), 1) def tearDown(self): - stats.close_spider(self.spider, '') + self.crawler.stats.close_spider(self.spider, '') diff --git a/scrapy/tests/test_mail.py b/scrapy/tests/test_mail.py index 4512997f8..f4e6a10d8 100644 --- a/scrapy/tests/test_mail.py +++ b/scrapy/tests/test_mail.py @@ -1,22 +1,22 @@ from cStringIO import StringIO import unittest -from scrapy.xlib.pydispatch import dispatcher - from scrapy.mail import MailSender, mail_sent +from scrapy.utils.test import get_crawler class MailSenderTest(unittest.TestCase): def setUp(self): self.catched_msg = None - dispatcher.connect(self._catch_mail_sent, signal=mail_sent) + self.crawler = get_crawler() + self.crawler.signals.connect(self._catch_mail_sent, signal=mail_sent) def tearDown(self): - dispatcher.disconnect(self._catch_mail_sent, signal=mail_sent) + self.crawler.signals.disconnect(self._catch_mail_sent, signal=mail_sent) def test_send(self): - mailsender = MailSender(debug=True) + mailsender = MailSender(debug=True, crawler=self.crawler) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body') assert self.catched_msg @@ -36,7 +36,7 @@ class MailSenderTest(unittest.TestCase): attach.seek(0) attachs = [('attachment', 'text/plain', attach)] - mailsender = MailSender(debug=True) + mailsender = MailSender(debug=True, crawler=self.crawler) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', attachs=attachs) diff --git a/scrapy/tests/test_spidermiddleware_depth.py b/scrapy/tests/test_spidermiddleware_depth.py index 6f9cab08a..beace38f9 100644 --- a/scrapy/tests/test_spidermiddleware_depth.py +++ b/scrapy/tests/test_spidermiddleware_depth.py @@ -4,6 +4,7 @@ from scrapy.contrib.spidermiddleware.depth import DepthMiddleware from scrapy.http import Response, Request from scrapy.spider import BaseSpider from scrapy.statscol import StatsCollector +from scrapy.utils.test import get_crawler class TestDepthMiddleware(TestCase): @@ -11,7 +12,7 @@ class TestDepthMiddleware(TestCase): def setUp(self): self.spider = BaseSpider('scrapytest.org') - self.stats = StatsCollector() + self.stats = StatsCollector(get_crawler()) self.stats.open_spider(self.spider) self.mw = DepthMiddleware(1, self.stats, True) diff --git a/scrapy/tests/test_stats.py b/scrapy/tests/test_stats.py index a39c0bd0e..f32af4208 100644 --- a/scrapy/tests/test_stats.py +++ b/scrapy/tests/test_stats.py @@ -1,18 +1,19 @@ import unittest from scrapy.spider import BaseSpider -from scrapy.xlib.pydispatch import dispatcher from scrapy.statscol import StatsCollector, DummyStatsCollector from scrapy.signals import stats_spider_opened, stats_spider_closing, \ stats_spider_closed +from scrapy.utils.test import get_crawler class StatsCollectorTest(unittest.TestCase): def setUp(self): + self.crawler = get_crawler() self.spider = BaseSpider('foo') def test_collector(self): - stats = StatsCollector() + stats = StatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) self.assertEqual(stats.get_value('anything'), None) self.assertEqual(stats.get_value('anything', 'default'), 'default') @@ -39,7 +40,7 @@ class StatsCollectorTest(unittest.TestCase): self.assertEqual(stats.get_value('test4'), 7) def test_dummy_collector(self): - stats = DummyStatsCollector() + stats = DummyStatsCollector(self.crawler) self.assertEqual(stats.get_stats(), {}) self.assertEqual(stats.get_value('anything'), None) self.assertEqual(stats.get_value('anything', 'default'), 'default') @@ -70,11 +71,11 @@ class StatsCollectorTest(unittest.TestCase): assert spider_stats == {'test': 1} signals_catched.add(stats_spider_closed) - dispatcher.connect(spider_opened, signal=stats_spider_opened) - dispatcher.connect(spider_closing, signal=stats_spider_closing) - dispatcher.connect(spider_closed, signal=stats_spider_closed) + self.crawler.signals.connect(spider_opened, signal=stats_spider_opened) + self.crawler.signals.connect(spider_closing, signal=stats_spider_closing) + self.crawler.signals.connect(spider_closed, signal=stats_spider_closed) - stats = StatsCollector() + stats = StatsCollector(self.crawler) stats.open_spider(self.spider) stats.set_value('test', 1, spider=self.spider) self.assertEqual([(self.spider, {'test': 1})], list(stats.iter_spider_stats())) @@ -83,9 +84,9 @@ class StatsCollectorTest(unittest.TestCase): assert stats_spider_closing in signals_catched assert stats_spider_closed in signals_catched - dispatcher.disconnect(spider_opened, signal=stats_spider_opened) - dispatcher.disconnect(spider_closing, signal=stats_spider_closing) - dispatcher.disconnect(spider_closed, signal=stats_spider_closed) + self.crawler.signals.disconnect(spider_opened, signal=stats_spider_opened) + self.crawler.signals.disconnect(spider_closing, signal=stats_spider_closing) + self.crawler.signals.disconnect(spider_closed, signal=stats_spider_closed) if __name__ == "__main__": unittest.main() diff --git a/scrapy/webservice.py b/scrapy/webservice.py index 0cca1b1b0..889dce5ad 100644 --- a/scrapy/webservice.py +++ b/scrapy/webservice.py @@ -6,7 +6,6 @@ See docs/topics/webservice.rst from twisted.web import server, error -from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy import log, signals from scrapy.utils.jsonrpc import jsonrpc_server_call @@ -80,8 +79,8 @@ class WebService(server.Site): root.putChild(res.ws_name, res) server.Site.__init__(self, root, logPath=logfile) self.noisy = False - dispatcher.connect(self.start_listening, signals.engine_started) - dispatcher.connect(self.stop_listening, signals.engine_stopped) + crawler.signals.connect(self.start_listening, signals.engine_started) + crawler.signals.connect(self.stop_listening, signals.engine_stopped) @classmethod def from_crawler(cls, crawler): From babfc6e79bfadab32b9bb4af477a0bf157774e37 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 28 Aug 2012 18:31:03 -0300 Subject: [PATCH 2/2] Updated documentation after singleton removal changes. Also removed some unused code and made some minor additional refactoring. --- docs/experimental/djangoitems.rst | 3 +- docs/index.rst | 6 +- docs/news.rst | 4 + docs/topics/api.rst | 333 +++++++++++++++++++++++ docs/topics/architecture.rst | 3 +- docs/topics/email.rst | 2 +- docs/topics/exporters.rst | 10 +- docs/topics/extensions.rst | 144 ++++------ docs/topics/firebug.rst | 4 +- docs/topics/item-pipeline.rst | 58 ++-- docs/topics/request-response.rst | 6 +- docs/topics/settings.rst | 178 ++---------- docs/topics/signals.rst | 7 +- docs/topics/stats.rst | 131 ++------- docs/topics/telnetconsole.rst | 16 +- scrapy/contrib/spidermiddleware/depth.py | 4 - scrapy/settings/default_settings.py | 1 - scrapy/tests/test_engine.py | 6 +- 18 files changed, 490 insertions(+), 426 deletions(-) create mode 100644 docs/topics/api.rst diff --git a/docs/experimental/djangoitems.rst b/docs/experimental/djangoitems.rst index 9560c9e47..c0ff82236 100644 --- a/docs/experimental/djangoitems.rst +++ b/docs/experimental/djangoitems.rst @@ -77,8 +77,7 @@ As said before, we can add other fields to the item:: p['age'] = '22' p['sex'] = 'M' -.. note:: fields added to the item won't be taken into account when doing a - :meth:`~DjangoItem.save` +.. note:: fields added to the item won't be taken into account when doing a :meth:`~DjangoItem.save` And we can override the fields of the model with your own:: diff --git a/docs/index.rst b/docs/index.rst index 317ccb2e1..61b053c9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -176,6 +176,7 @@ Extending Scrapy topics/downloader-middleware topics/spider-middleware topics/extensions + topics/api :doc:`topics/architecture` Understand the Scrapy architecture. @@ -187,9 +188,10 @@ Extending Scrapy Customize the input and output of your spiders. :doc:`topics/extensions` - Add any custom functionality using :doc:`signals ` and the - Scrapy API + Extend Scrapy with your custom functionality +:doc:`topics/api` + Use it on extensions and middlewares to extend Scrapy functionality Reference ========= diff --git a/docs/news.rst b/docs/news.rst index 8b4fdc1b0..426e37fd9 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6,6 +6,9 @@ Release notes Scrapy changes: +- dropped Signals singleton. Signals should now be accesed through the Crawler.signals attribute. See the signals documentation for more info. +- dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. +- documented :ref:`topics-api` - `lxml` is now the default selectors backend instead of `libxml2` - ported FormRequest.from_response() to use `lxml`_ instead of `ClientForm`_ - removed modules: ``scrapy.xlib.BeautifulSoup`` and ``scrapy.xlib.ClientForm`` @@ -21,6 +24,7 @@ Scrapy changes: - removed per-spider settings (to be replaced by instantiating multiple crawler objects) - ``USER_AGENT`` spider attribute will no longer work, use ``user_agent`` attribute instead - ``DOWNLOAD_TIMEOUT`` spider attribute will no longer work, use ``download_timeout`` attribute instead +- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library Scrapyd changes: diff --git a/docs/topics/api.rst b/docs/topics/api.rst new file mode 100644 index 000000000..b079bcf5b --- /dev/null +++ b/docs/topics/api.rst @@ -0,0 +1,333 @@ +.. _topics-api: + +======== +Core API +======== + +.. versionadded:: 0.15 + +This section documents the Scrapy core API, and it's intended for developers of +extensions and middlewares. + +.. _topics-api-crawler: + +Crawler API +=========== + +The main entry point to Scrapy API is the :class:`~scrapy.crawler.Crawler` +object, passed to extensions through the ``from_crawler`` class method. This +object provides access to all Scrapy core components, and it's the only way for +extensions to access them and hook their functionality into Scrapy. + +.. module:: scrapy.crawler + :synopsis: The Scrapy crawler + +The Extension Manager is responsible for loading and keeping track of installed +extensions and it's configured through the :setting:`EXTENSIONS` setting which +contains a dictionary of all available extensions and their order similar to +how you :ref:`configure the downloader middlewares +`. + +.. class:: Crawler(settings) + + The Crawler object must be instantiated with a + :class:`scrapy.settings.Settings` object. + + .. attribute:: settings + + The settings manager of this crawler. + + This is used by extensions & middlewares to access the Scrapy settings + of this crawler. + + For an introduction on Scrapy settings see :ref:`topics-settings`. + + For the API see :class:`~scrapy.settings.Settings` class. + + .. attribute:: signals + + The signals manager of this crawler. + + This is used by extensions & middlewares to hook themselves into Scrapy + functionality. + + For an introduction on signals see :ref:`topics-signals`. + + For the API see :class:`~scrapy.signalmanager.SignalManager` class. + + .. attribute:: stats + + The stats collector of this crawler. + + This is used from extensions & middlewares to record stats of their + behaviour, or access stats collected by other extensions. + + For an introduction on stats collection see :ref:`topics-stats`. + + For the API see :class:`~scrapy.statscol.StatsCollector` class. + + .. attribute:: extensions + + The extension manager that keeps track of enabled extensions. + + Most extensions won't need to access this attribute. + + For an introduction on extensions and a list of available extensions on + Scrapy see :ref:`topics-extensions`. + + .. attribute:: spiders + + The spider manager which takes care of loading and instantiating + spiders. + + Most extensions won't need to access this attribute. + + .. attribute:: engine + + The execution engine, which coordinates the core crawling logic + between the scheduler, downloader and spiders. + + Some extension may want to access the Scrapy engine, to modify inspect + or modify the downloader and scheduler behaviour, although this is an + advanced use and this API is not yet stable. + + .. method:: configure() + + Configure the crawler. + + This loads extensions, middlewares and spiders, leaving the crawler + ready to be started. It also configures the execution engine. + + .. method:: start() + + Start the crawler. This calss :meth:`configure` if it hasn't been called yet. + +Settings API +============ + +.. module:: scrapy.settings + :synopsis: Settings manager + +.. class:: Settings() + + This object that provides access to Scrapy settings. + + .. attribute:: overrides + + Global overrides are the ones that take most precedence, and are usually + populated by command-line options. + + Overrides should be populated *before* configuring the Crawler object + (through the :meth:`~scrapy.crawler.Crawler.configure` method), + otherwise they won't have any effect. You don't typically need to worry + about overrides unless you are implementing your own Scrapy command. + + .. method:: get(name, default=None) + + Get a setting value without affecting its original type. + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + + .. method:: getbool(name, default=False) + + Get a setting value as a boolean. For example, both ``1`` and ``'1'``, and + ``True`` return ``True``, while ``0``, ``'0'``, ``False`` and ``None`` + return ``False```` + + For example, settings populated through environment variables set to ``'0'`` + will return ``False`` when using this method. + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + + .. method:: getint(name, default=0) + + Get a setting value as an int + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + + .. method:: getfloat(name, default=0.0) + + Get a setting value as a float + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + + .. method:: getlist(name, default=None) + + Get a setting value as a list. If the setting original type is a list it + will be returned verbatim. If it's a string it will be split by ",". + + For example, settings populated through environment variables set to + ``'one,two'`` will return a list ['one', 'two'] when using this method. + + :param name: the setting name + :type name: string + + :param default: the value to return if no setting is found + :type default: any + +.. _topics-api-signals: + +Signals API +=========== + +.. module:: scrapy.signalmanager + :synopsis: The signal manager + +.. class:: SignalManager + + .. method:: connect(receiver, signal) + + Connect a receiver function to a signal. + + The signal can be any object, although Scrapy comes with some + predefined signals that are documented in the :ref:`topics-signals` + section. + + :param receiver: the function to be connected + :type receiver: callable + + :param signal: the signal to connect to + :type signal: object + + .. method:: send_catch_log(signal, \*\*kwargs) + + Send a signal, catch exceptions and log them. + + The keyword arguments are passed to the signal handlers (connected + through the :meth:`connect` method). + + .. method:: send_catch_log_deferred(signal, \*\*kwargs) + + Like :meth:`send_catch_log` but supports returning `deferreds`_ from + signal handlers. + + Returns a `deferred`_ that gets fired once all signal handlers + deferreds were fired. Send a signal, catch exceptions and log them. + + The keyword arguments are passed to the signal handlers (connected + through the :meth:`connect` method). + + .. method:: disconnect(receiver, signal) + + Disconnect a receiver function from a signal. This has the opposite + effect of the :meth:`connect` method, and the arguments are the same. + + .. method:: disconnect_all(signal) + + Disconnect all receivers from the given signal. + + :param signal: the signal to disconnect from + :type signal: object + +.. _topics-api-stats: + +Stats Collector API +=================== + +There are several Stats Collectors available under the +:mod:`scrapy.statscol` module and they all implement the Stats +Collector API defined by the :class:`~scrapy.statscol.StatsCollector` +class (which they all inherit from). + +.. module:: scrapy.statscol + :synopsis: Stats Collectors + +.. class:: StatsCollector + + .. method:: get_value(key, default=None, spider=None) + + Return the value for the given stats key or default if it doesn't exist. + If spider is ``None`` the global stats table is consulted, otherwise the + spider specific one is. If the spider is not yet opened a ``KeyError`` + exception is raised. + + .. method:: get_stats(spider=None) + + Get all stats from the given spider (if spider is given) or all global + stats otherwise, as a dict. If spider is not opened ``KeyError`` is + raised. + + .. method:: set_value(key, value, spider=None) + + Set the given value for the given stats key on the global stats (if + spider is not given) or the spider-specific stats (if spider is given), + which must be opened or a ``KeyError`` will be raised. + + .. method:: set_stats(stats, spider=None) + + Set the given stats (as a dict) for the given spider. If the spider is + not opened a ``KeyError`` will be raised. + + .. method:: inc_value(key, count=1, start=0, spider=None) + + Increment the value of the given stats key, by the given count, + assuming the start value given (when it's not set). If spider is not + given the global stats table is used, otherwise the spider-specific + stats table is used, which must be opened or a ``KeyError`` will be + raised. + + .. method:: max_value(key, value, spider=None) + + Set the given value for the given key only if current value for the + same key is lower than value. If there is no current value for the + given key, the value is always set. If spider is not given, the global + stats table is used, otherwise the spider-specific stats table is used, + which must be opened or a KeyError will be raised. + + .. method:: min_value(key, value, spider=None) + + Set the given value for the given key only if current value for the + same key is greater than value. If there is no current value for the + given key, the value is always set. If spider is not given, the global + stats table is used, otherwise the spider-specific stats table is used, + which must be opened or a KeyError will be raised. + + .. method:: clear_stats(spider=None) + + Clear all global stats (if spider is not given) or all spider-specific + stats if spider is given, in which case it must be opened or a + ``KeyError`` will be raised. + + .. method:: iter_spider_stats() + + Return a iterator over ``(spider, spider_stats)`` for each open spider + currently tracked by the stats collector, where ``spider_stats`` is the + dict containing all spider-specific stats. + + Global stats are not included in the iterator. If you want to get + those, use :meth:`get_stats` method. + + The following methods are not part of the stats collection api but instead + used when implementing custom stats collectors: + + .. method:: open_spider(spider) + + Open the given spider for stats collection. + + .. method:: close_spider(spider) + + Close the given spider. After this is called, no more specific stats + for this spider can be accessed. + + .. method:: engine_stopped() + + Called after the engine is stopped, to dump or persist global stats. + +.. _deferreds: http://twistedmatrix.com/documents/current/core/howto/defer.html +.. _deferred: http://twistedmatrix.com/documents/current/core/howto/defer.html diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index bdab8552c..a3c462702 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -80,7 +80,8 @@ functionality by plugging custom code. For more information see Data flow ========= -The data flow in Scrapy is controlled by the Engine, and goes like this: +The data flow in Scrapy is controlled by the execution engine, and goes like +this: 1. The Engine opens a domain, locates the Spider that handles that domain, and asks the spider for the first URLs to crawl. diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 9c01c9c8d..a6a54445f 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -12,7 +12,7 @@ library, Scrapy provides its own facility for sending e-mails which is very easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid interfering with the non-blocking IO of the crawler. It also provides a simple API for sending attachments and it's very easy to configure, with a few -:ref:`settings `. .. _smtplib: http://docs.python.org/library/smtplib.html .. _Twisted non-blocking IO: http://twistedmatrix.com/projects/core/documentation/howto/async.html diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 4090b4ed8..7c8d93d11 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -39,17 +39,21 @@ the end of the exporting process Here you can see an :doc:`Item Pipeline ` which uses an Item Exporter to export scraped items to different files, one per spider:: - from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.contrib.exporter import XmlItemExporter class XmlExportPipeline(object): def __init__(self): - dispatcher.connect(self.spider_opened, signals.spider_opened) - dispatcher.connect(self.spider_closed, signals.spider_closed) self.files = {} + @classmethod + def from_crawler(cls, crawler): + pipeline = cls() + crawler.signals.connect(pipeline.spider_opened, signals.spider_opened) + crawler.signals.connect(pipeline.spider_closed, signals.spider_closed) + return pipeline + def spider_opened(self, spider): file = open('%s_products.xml' % spider.name, 'w+b') self.files[spider] = file diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 212043988..037c4362b 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -62,113 +62,81 @@ Not all available extensions will be enabled. Some of them usually depend on a particular setting. For example, the HTTP Cache extension is available by default but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set. -Accessing enabled extensions -============================ - -Even though it's not usually needed, you can access extension objects through -the :ref:`topics-extensions-ref-manager` which is populated when extensions are -loaded. For example, to access the ``WebService`` extension:: - - from scrapy.project import extensions - webservice_extension = extensions.enabled['WebService'] - -.. see also:: - - :ref:`topics-extensions-ref-manager`, for the complete Extension Manager - reference. - Writing your own extension ========================== Writing your own extension is easy. Each extension is a single Python class which doesn't need to implement any particular method. -All extension initialization code must be performed in the class constructor -(``__init__`` method). If that method raises the +The main entry point for a Scrapy extension (this also includes middlewares and +pipelines) is the ``from_crawler`` class method which receives a +``Crawler`` instance which is the main object controlling the Scrapy crawler. +Through that object you can access settings, signals, stats, and also control +the crawler behaviour, if your extension needs to such thing. + +Typically, extensions connect to :ref:`signals ` and perform +tasks triggered by them. + +Finally, if the ``from_crawler`` method raises the :exc:`~scrapy.exceptions.NotConfigured` exception, the extension will be disabled. Otherwise, the extension will be enabled. -Let's take a look at the following example extension which just logs a message -every time a domain/spider is opened and closed:: +Sample extension +---------------- + +Here we will implement a simple extension to illustrate the concepts described +in the previous section. This extension will log a message every time: + +* a spider is opened +* a spider is closed +* a specific number of items are scraped + +The extension will be enabled through the ``MYEXT_ENABLED`` setting and the +number of items will be specified through the ``MYEXT_ITEMCOUNT`` setting. + +Here is the code of such extension:: - from scrapy.xlib.pydispatch import dispatcher from scrapy import signals + from scrapy.exceptions import NotConfigured class SpiderOpenCloseLogging(object): - def __init__(self): - dispatcher.connect(self.spider_opened, signal=signals.spider_opened) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + def __init__(self, item_count): + self.item_count = item_count + self.items_scraped = 0 + + @classmethod + def from_crawler(cls, crawler): + # first check if the extension should be enabled and raise + # NotConfigured otherwise + if not crawler.settings.getbool('MYEXT_ENABLED'): + raise NotConfigured + + # get the number of items from settings + item_count = crawler.settings.getint('MYEXT_ITEMCOUNT', 1000) + + # instantiate the extension object + ext = cls(item_count) + + # connect the extension object to signals + crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened) + crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed) + crawler.signals.connect(ext.item_scraped, signal=signals.item_scraped) + + # return the extension object + return ext def spider_opened(self, spider): - log.msg("opened spider %s" % spider.name) + spider.log("opened spider %s" % spider.name) def spider_closed(self, spider): - log.msg("closed spider %s" % spider.name) - - -.. _topics-extensions-ref-manager: - -Extension Manager -================= - -.. module:: scrapy.extension - :synopsis: The extension manager - -The Extension Manager is responsible for loading and keeping track of installed -extensions and it's configured through the :setting:`EXTENSIONS` setting which -contains a dictionary of all available extensions and their order similar to -how you :ref:`configure the downloader middlewares -`. - -.. class:: ExtensionManager - - The Extension Manager is a singleton object, which is instantiated at module - loading time and can be accessed like this:: - - from scrapy.project import extensions - - .. attribute:: loaded - - A boolean which is True if extensions are already loaded or False if - they're not. - - .. attribute:: enabled - - A dict with the enabled extensions. The keys are the extension class names, - and the values are the extension objects. Example:: - - >>> from scrapy.project import extensions - >>> extensions.load() - >>> print extensions.enabled - {'CoreStats': , - 'WebService': , - ... - - .. attribute:: disabled - - A dict with the disabled extensions. The keys are the extension class names, - and the values are the extension class paths (because objects are never - instantiated for disabled extensions). Example:: - - >>> from scrapy.project import extensions - >>> extensions.load() - >>> print extensions.disabled - {'MemoryDebugger': 'scrapy.contrib.memdebug.MemoryDebugger', - 'MyExtension': 'myproject.extensions.MyExtension', - ... - - .. method:: load() - - Load the available extensions configured in the :setting:`EXTENSIONS` - setting. On a standard run, this method is usually called by the Execution - Manager, but you may need to call it explicitly if you're dealing with - code outside Scrapy. - - .. method:: reload() - - Reload the available extensions. See :meth:`load`. + spider.log("closed spider %s" % spider.name) + def item_scrapde(self, item, spider): + self.items_scraped += 1 + if self.items_scraped == self.item_count: + spider.log("scraped %d items, resetting counter" % self.items_scraped) + self.item_count = 0 .. _topics-extensions-ref: diff --git a/docs/topics/firebug.rst b/docs/topics/firebug.rst index 98d6da3a2..395f3bfe1 100644 --- a/docs/topics/firebug.rst +++ b/docs/topics/firebug.rst @@ -7,8 +7,8 @@ Using Firebug for scraping .. note:: Google Directory, the example website used in this guide is no longer available as it `has been shut down by Google`_. The concepts in this guide are still valid though. If you want to update this guide to use a new - (working) site, your contribution will be more than welcome!. See - :ref:`topics-contributing` for information on how to do so. + (working) site, your contribution will be more than welcome!. See :ref:`topics-contributing` + for information on how to do so. Introduction ============ diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 7ebca5c5f..8714971bb 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -104,6 +104,30 @@ format:: item pipelines. If you really want to store all scraped items into a JSON file you should use the :ref:`Feed exports `. +Duplicates filter +----------------- + +A filter that looks for duplicate items, and drops those items that were +already processed. Let say that our items have an unique id, but our spider +returns multiples items with the same id:: + + + from scrapy import signals + from scrapy.exceptions import DropItem + + class DuplicatesPipeline(object): + + def __init__(self): + self.ids_seen = set() + + def process_item(self, item, spider): + if item['id'] in self.ids_seen: + raise DropItem("Duplicate item found: %s" % item) + else: + self.ids_seen.add(item['id']) + return item + + Activating an Item Pipeline component ===================================== @@ -114,37 +138,3 @@ To activate an Item Pipeline component you must add its class to the 'myproject.pipeline.PricePipeline', 'myproject.pipeline.JsonWriterPipeline', ] - -Item pipeline example with resources per spider -=============================================== - -Sometimes you need to keep resources about the items processed grouped per -spider, and delete those resource when a spider finishes. - -An example is a filter that looks for duplicate items, and drops those items -that were already processed. Let say that our items have an unique id, but our -spider returns multiples items with the same id:: - - - from scrapy.xlib.pydispatch import dispatcher - from scrapy import signals - from scrapy.exceptions import DropItem - - class DuplicatesPipeline(object): - def __init__(self): - self.duplicates = {} - dispatcher.connect(self.spider_opened, signals.spider_opened) - dispatcher.connect(self.spider_closed, signals.spider_closed) - - def spider_opened(self, spider): - self.duplicates[spider] = set() - - def spider_closed(self, spider): - del self.duplicates[spider] - - def process_item(self, item, spider): - if item['id'] in self.duplicates[spider]: - raise DropItem("Duplicate item found: %s" % item) - else: - self.duplicates[spider].add(item['id']) - return item diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ec02a50e8..faead36be 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -51,11 +51,13 @@ Request objects (for single valued headers) or lists (for multi-valued headers). :type headers: dict - :param cookies: the request cookies. These can be sent in two forms:: + :param cookies: the request cookies. These can be sent in two forms. + + 1. Using a dict:: request_with_cookies = Request(url="http://www.example.com", cookies={'currency': 'USD', 'country': 'UY'}) - :: + 2. Using a list of dicts:: request_with_cookies = Request(url="http://www.example.com", cookies=[{'name': 'currency', diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 682206db8..5a8be53ff 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -4,9 +4,6 @@ Settings ======== -.. module:: scrapy.conf - :synopsis: Settings manager - The Scrapy settings allows you to customize the behaviour of all Scrapy components, including the core, extensions, pipelines and spiders themselves. @@ -49,14 +46,11 @@ These mechanisms are described in more detail below. ------------------- Global overrides are the ones that take most precedence, and are usually -populated by command-line options. +populated by command-line options. You can also override one (or more) settings +from command line using the ``-s`` (or ``--set``) command line option. -Example:: - >>> from scrapy.conf import settings - >>> settings.overrides['LOG_ENABLED'] = True - -You can also override one (or more) settings from command line using the -``-s`` (or ``--set``) command line option. +For more information see the :attr:`~scrapy.settings.Settings.overrides` +Settings attribute. .. highlight:: sh @@ -90,82 +84,22 @@ How to access settings .. highlight:: python -Here's an example of the simplest way to access settings from Python code:: +Settings can be accessed through the :attr:`scrapy.crawler.Crawler.settings` +attribute of the Crawler that is passed to ``from_crawler`` method in +extensions and middlewares:: - >>> from scrapy.conf import settings - >>> print settings['LOG_ENABLED'] - True + class MyExtension(object): + + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + if settings['LOG_ENABLED']: + print "log is enabled!" In other words, settings can be accesed like a dict, but it's usually preferred to extract the setting in the format you need it to avoid type errors. In order -to do that you'll have to use one of the following methods: - -.. class:: Settings() - - There is a (singleton) Settings object automatically instantiated when the - :mod:`scrapy.conf` module is loaded, and it's usually accessed like this:: - - >>> from scrapy.conf import settings - - .. method:: get(name, default=None) - - Get a setting value without affecting its original type. - - :param name: the setting name - :type name: string - - :param default: the value to return if no setting is found - :type default: any - - .. method:: getbool(name, default=False) - - Get a setting value as a boolean. For example, both ``1`` and ``'1'``, and - ``True`` return ``True``, while ``0``, ``'0'``, ``False`` and ``None`` - return ``False```` - - For example, settings populated through environment variables set to ``'0'`` - will return ``False`` when using this method. - - :param name: the setting name - :type name: string - - :param default: the value to return if no setting is found - :type default: any - - .. method:: getint(name, default=0) - - Get a setting value as an int - - :param name: the setting name - :type name: string - - :param default: the value to return if no setting is found - :type default: any - - .. method:: getfloat(name, default=0.0) - - Get a setting value as a float - - :param name: the setting name - :type name: string - - :param default: the value to return if no setting is found - :type default: any - - .. method:: getlist(name, default=None) - - Get a setting value as a list. If the setting original type is a list it - will be returned verbatim. If it's a string it will be split by ",". - - For example, settings populated through environment variables set to - ``'one,two'`` will return a list ['one', 'two'] when using this method. - - :param name: the setting name - :type name: string - - :param default: the value to return if no setting is found - :type default: any - +to do that you'll have to use one of the methods provided the +:class:`~scrapy.settings.Settings` API. Rationale for setting names =========================== @@ -477,78 +411,17 @@ The class used to detect and filter duplicate requests. The default (``RFPDupeFilter``) filters based on request fingerprint using the ``scrapy.utils.request.request_fingerprint`` function. -.. setting:: EDITOR +.. setting:: jDITOR EDITOR ------ +Default: `depends on the environment` + The editor to use for editing spiders with the :command:`edit` command. It defaults to the ``EDITOR`` environment variable, if set. Otherwise, it defaults to ``vi`` (on Unix systems) or the IDLE editor (on Windows). -.. setting:: ENCODING_ALIASES - -ENCODING_ALIASES ----------------- - -Default: ``{}`` - -A mapping of custom encoding aliases for your project, where the keys are the -aliases (and must be lower case) and the values are the encodings they map to. - -This setting extends the :setting:`ENCODING_ALIASES_BASE` setting which -contains some default mappings. - -.. setting:: ENCODING_ALIASES_BASE - -ENCODING_ALIASES_BASE ---------------------- - -Default:: - - { - # gb2312 is superseded by gb18030 - 'gb2312': 'gb18030', - 'chinese': 'gb18030', - 'csiso58gb231280': 'gb18030', - 'euc- cn': 'gb18030', - 'euccn': 'gb18030', - 'eucgb2312-cn': 'gb18030', - 'gb2312-1980': 'gb18030', - 'gb2312-80': 'gb18030', - 'iso- ir-58': 'gb18030', - # gbk is superseded by gb18030 - 'gbk': 'gb18030', - '936': 'gb18030', - 'cp936': 'gb18030', - 'ms936': 'gb18030', - # latin_1 is a subset of cp1252 - 'latin_1': 'cp1252', - 'iso-8859-1': 'cp1252', - 'iso8859-1': 'cp1252', - '8859': 'cp1252', - 'cp819': 'cp1252', - 'latin': 'cp1252', - 'latin1': 'cp1252', - 'l1': 'cp1252', - # others - 'zh-cn': 'gb18030', - 'win-1251': 'cp1251', - 'macintosh' : 'mac_roman', - 'x-sjis': 'shift_jis', - } - -The default encoding aliases defined in Scrapy. Don't override this setting in -your project, override :setting:`ENCODING_ALIASES` instead. - -The reason why `ISO-8859-1`_ (and all its aliases) are mapped to `CP1252`_ is -due to a well known browser hack. For more information see: `Character -encodings in HTML`_. - -.. _ISO-8859-1: http://en.wikipedia.org/wiki/ISO/IEC_8859-1 -.. _CP1252: http://en.wikipedia.org/wiki/Windows-1252 -.. _Character encodings in HTML: http://en.wikipedia.org/wiki/Character_encodings_in_HTML - .. setting:: EXTENSIONS EXTENSIONS @@ -880,8 +753,8 @@ STATS_CLASS Default: ``'scrapy.statscol.MemoryStatsCollector'`` -The class to use for collecting stats (must implement the Stats Collector API, -or subclass the StatsCollector class). +The class to use for collecting stats, who must implement the +:ref:`topics-api-stats`. .. setting:: STATS_DUMP @@ -896,15 +769,6 @@ closed, while the global stats are dumped when the Scrapy process finishes. For more info see: :ref:`topics-stats`. -.. setting:: STATS_ENABLED - -STATS_ENABLED -------------- - -Default: ``True`` - -Enable stats collection. - .. setting:: STATSMAILER_RCPTS STATSMAILER_RCPTS diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f1eb00385..81e1eb2b6 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -13,11 +13,8 @@ Even though signals provide several arguments, the handlers that catch them don't need to accept all of them - the signal dispatching mechanism will only deliver the arguments that the handler receives. -Finally, for more detailed information about signals internals see the -documentation of `pydispatcher`_ (the which the signal dispatching mechanism is -based on). - -.. _pydispatcher: http://pydispatcher.sourceforge.net/ +You can connect to signals (or send your own) through the +:ref:`topics-api-signals`. Deferred signal handlers ======================== diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 162d77f90..1a8ec56ea 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -4,16 +4,11 @@ Stats Collection ================ -Overview -======== - -Scrapy provides a convenient service for collecting stats in the form of +Scrapy provides a convenient facility for collecting stats in the form of key/values, both globally and per spider. It's called the Stats Collector, and -it's a singleton which can be imported and used quickly, as illustrated by the -examples in the :ref:`topics-stats-usecases` section below. - -The stats collection is enabled by default but can be disabled through the -:setting:`STATS_ENABLED` setting. +can be accesed through the :attr:`~scrapy.crawler.Crawler.stats` attribute of +the :ref:`topics-api-crawler`, as illustrated by the examples in the +:ref:`topics-stats-usecases` section below. However, the Stats Collector is always available, so you can always import it in your module and use its API (to increment or set new stat keys), regardless @@ -36,9 +31,12 @@ the spider is closed. Common Stats Collector uses =========================== -Import the stats collector:: +Access the stats collector throught the :attr:`~scrapy.crawler.Crawler.stats` +attribute:: - from scrapy.stats import stats + @classmethod + def from_crawler(cls, crawler): + stats = crawler.stats Set global stat value:: @@ -66,8 +64,7 @@ Get all global stats (ie. not particular to any spider):: >>> stats.get_stats() {'hostname': 'localhost', 'spiders_crawled': 8} -Set spider specific stat value (spider stats must be opened first, but this -task is handled automatically by the Scrapy engine):: +Set spider specific stat value:: stats.set_value('start_time', datetime.now(), spider=some_spider) @@ -95,100 +92,6 @@ Get all stats from a given spider:: >>> stats.get_stats(spider=some_spider) {'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} -.. _topics-stats-ref: - -Stats Collector API -=================== - -There are several Stats Collectors available under the -:mod:`scrapy.statscol` module and they all implement the Stats -Collector API defined by the :class:`~scrapy.statscol.StatsCollector` -class (which they all inherit from). - -.. module:: scrapy.statscol - :synopsis: Basic Stats Collectors - -.. class:: StatsCollector - - .. method:: get_value(key, default=None, spider=None) - - Return the value for the given stats key or default if it doesn't exist. - If spider is ``None`` the global stats table is consulted, otherwise the - spider specific one is. If the spider is not yet opened a ``KeyError`` - exception is raised. - - .. method:: get_stats(spider=None) - - Get all stats from the given spider (if spider is given) or all global - stats otherwise, as a dict. If spider is not opened ``KeyError`` is - raised. - - .. method:: set_value(key, value, spider=None) - - Set the given value for the given stats key on the global stats (if - spider is not given) or the spider-specific stats (if spider is given), - which must be opened or a ``KeyError`` will be raised. - - .. method:: set_stats(stats, spider=None) - - Set the given stats (as a dict) for the given spider. If the spider is - not opened a ``KeyError`` will be raised. - - .. method:: inc_value(key, count=1, start=0, spider=None) - - Increment the value of the given stats key, by the given count, - assuming the start value given (when it's not set). If spider is not - given the global stats table is used, otherwise the spider-specific - stats table is used, which must be opened or a ``KeyError`` will be - raised. - - .. method:: max_value(key, value, spider=None) - - Set the given value for the given key only if current value for the - same key is lower than value. If there is no current value for the - given key, the value is always set. If spider is not given, the global - stats table is used, otherwise the spider-specific stats table is used, - which must be opened or a KeyError will be raised. - - .. method:: min_value(key, value, spider=None) - - Set the given value for the given key only if current value for the - same key is greater than value. If there is no current value for the - given key, the value is always set. If spider is not given, the global - stats table is used, otherwise the spider-specific stats table is used, - which must be opened or a KeyError will be raised. - - .. method:: clear_stats(spider=None) - - Clear all global stats (if spider is not given) or all spider-specific - stats if spider is given, in which case it must be opened or a - ``KeyError`` will be raised. - - .. method:: iter_spider_stats() - - Return a iterator over ``(spider, spider_stats)`` for each open spider - currently tracked by the stats collector, where ``spider_stats`` is the - dict containing all spider-specific stats. - - Global stats are not included in the iterator. If you want to get - those, use :meth:`get_stats` method. - - .. method:: open_spider(spider) - - Open the given spider for stats collection. This method must be called - prior to working with any stats specific to that spider, but this task - is handled automatically by the Scrapy engine. - - .. method:: close_spider(spider) - - Close the given spider. After this is called, no more specific stats - for this spider can be accessed. This method is called automatically on - the :signal:`spider_closed` signal. - - .. method:: engine_stopped() - - Called after the engine is stopped, to dump or persist global stats. - Available Stats Collectors ========================== @@ -197,9 +100,8 @@ available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The default Stats Collector used is the :class:`MemoryStatsCollector`. -When stats are disabled (through the :setting:`STATS_ENABLED` setting) the -:setting:`STATS_CLASS` setting is ignored and the :class:`DummyStatsCollector` -is used. +.. module:: scrapy.statscol + :synopsis: Stats Collectors MemoryStatsCollector -------------------- @@ -223,9 +125,12 @@ DummyStatsCollector .. class:: DummyStatsCollector - A Stats collector which does nothing but is very efficient. This is the - Stats Collector used when stats are disabled (through the - :setting:`STATS_ENABLED` setting). + A Stats collector which does nothing but is very efficient (beacuse it does + nothing). This stats collector can be set via the :setting:`STATS_CLASS` + setting, to disable stats collect in order to improve performance. However, + the performance penalty of stats collection is usually marginal compared to + other Scrapy workload like parsing pages. + Stats signals ============= diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 4b8b9a9d8..795d731e3 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -43,21 +43,21 @@ convenience: +----------------+-------------------------------------------------------------------+ | Shortcut | Description | +================+===================================================================+ -| ``crawler`` | the Scrapy Crawler object (``scrapy.crawler``) | +| ``crawler`` | the Scrapy Crawler (:class:`scrapy.crawler.Crawler` object) | +----------------+-------------------------------------------------------------------+ -| ``engine`` | the Scrapy Engine object (``scrapy.core.engine``) | +| ``engine`` | Crawler.engine attribute | +----------------+-------------------------------------------------------------------+ -| ``spider`` | the spider object (only if there is a single spider opened) | +| ``spider`` | the active spider | +----------------+-------------------------------------------------------------------+ -| ``slot`` | the engine slot (only if there is a single spider opened) | +| ``slot`` | the engine slot | +----------------+-------------------------------------------------------------------+ -| ``extensions`` | the Extension Manager (``scrapy.project.crawler.extensions``) | +| ``extensions`` | the Extension Manager (Crawler.extensions attribute) | +----------------+-------------------------------------------------------------------+ -| ``stats`` | the Stats Collector (``scrapy.stats.stats``) | +| ``stats`` | the Stats Collector (Crawler.stats attribute) | +----------------+-------------------------------------------------------------------+ -| ``settings`` | the Scrapy settings object (``scrapy.conf.settings``) | +| ``settings`` | the Scrapy settings object (Crawler.settings attribute) | +----------------+-------------------------------------------------------------------+ -| ``est`` | print a report of the current engine status | +| ``est`` | print a report of the engine status | +----------------+-------------------------------------------------------------------+ | ``prefs`` | for memory debugging (see :ref:`topics-leaks`) | +----------------+-------------------------------------------------------------------+ diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index 438051cb5..2a218f990 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -4,11 +4,8 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ -import warnings - from scrapy import log from scrapy.http import Request -from scrapy.exceptions import ScrapyDeprecationWarning class DepthMiddleware(object): @@ -22,7 +19,6 @@ class DepthMiddleware(object): def from_crawler(cls, crawler): settings = crawler.settings maxdepth = settings.getint('DEPTH_LIMIT') - usestats = settings.getbool('DEPTH_STATS') verbose = settings.getbool('DEPTH_STATS_VERBOSE') prio = settings.getint('DEPTH_PRIORITY') return cls(maxdepth, crawler.stats, verbose, prio) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 3baf4cdb7..dd4b587ae 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -216,7 +216,6 @@ SPIDER_MIDDLEWARES_BASE = { SPIDER_MODULES = [] STATS_CLASS = 'scrapy.statscol.MemoryStatsCollector' -STATS_ENABLED = True STATS_DUMP = True STATSMAILER_RCPTS = [] diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 22ae44ead..da1788811 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -90,13 +90,13 @@ class CrawlerRun(object): for name, signal in vars(signals).items(): if not name.startswith('_'): dispatcher.connect(self.record_signal, signal) - dispatcher.connect(self.item_scraped, signals.item_scraped) - dispatcher.connect(self.request_received, signals.request_received) - dispatcher.connect(self.response_downloaded, signals.response_downloaded) self.crawler = get_crawler() self.crawler.install() self.crawler.configure() + self.crawler.signals.connect(self.item_scraped, signals.item_scraped) + self.crawler.signals.connect(self.request_received, signals.request_received) + self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) self.crawler.crawl(self.spider) self.crawler.start()