diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index 7dab6c55f..e07224e7a 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -13,12 +13,12 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy import signals, log from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.project import crawler from scrapy.conf import settings class CloseSpider(object): - def __init__(self): + def __init__(self, crawler): + self.crawler = crawler self.timeout = settings.getint('CLOSESPIDER_TIMEOUT') self.itemcount = settings.getint('CLOSESPIDER_ITEMCOUNT') # XXX: legacy support - remove for future releases @@ -43,28 +43,32 @@ class CloseSpider(object): dispatcher.connect(self.item_scraped, signal=signals.item_scraped) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + def catch_log(self, event): if event.get('logLevel') == log.ERROR: spider = event.get('spider') if spider: self.errorcounts[spider] += 1 if self.errorcounts[spider] == self.errorcount: - crawler.engine.close_spider(spider, 'closespider_errorcount') + self.crawler.engine.close_spider(spider, 'closespider_errorcount') def page_count(self, response, request, spider): self.pagecounts[spider] += 1 if self.pagecounts[spider] == self.pagecount: - crawler.engine.close_spider(spider, 'closespider_pagecount') + self.crawler.engine.close_spider(spider, 'closespider_pagecount') def spider_opened(self, spider): self.tasks[spider] = reactor.callLater(self.timeout, \ - crawler.engine.close_spider, spider=spider, \ + self.crawler.engine.close_spider, spider=spider, \ reason='closespider_timeout') def item_scraped(self, item, spider): self.counts[spider] += 1 if self.counts[spider] == self.itemcount: - crawler.engine.close_spider(spider, 'closespider_itemcount') + self.crawler.engine.close_spider(spider, 'closespider_itemcount') def spider_closed(self, spider): self.counts.pop(spider, None) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 220b5926a..bf64677f8 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -9,25 +9,28 @@ import robotparser from scrapy.xlib.pydispatch import dispatcher from scrapy import signals, log -from scrapy.project import crawler from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -from scrapy.conf import settings class RobotsTxtMiddleware(object): DOWNLOAD_PRIORITY = 1000 - def __init__(self): - if not settings.getbool('ROBOTSTXT_OBEY'): + def __init__(self, crawler): + if not crawler.settings.getbool('ROBOTSTXT_OBEY'): raise NotConfigured + self.crawler = crawler self._parsers = {} self._spider_netlocs = {} self._useragents = {} dispatcher.connect(self.spider_opened, signals.spider_opened) dispatcher.connect(self.spider_closed, signals.spider_closed) + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + def process_request(self, request, spider): useragent = self._useragents[spider] rp = self.robot_parser(request, spider) @@ -42,7 +45,7 @@ class RobotsTxtMiddleware(object): self._parsers[netloc] = None robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY) - dfd = crawler.engine.download(robotsreq, spider) + dfd = self.crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots) self._spider_netlocs[spider].add(netloc) return self._parsers[netloc] diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index e3e051ab5..95de44997 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -12,31 +12,34 @@ from twisted.internet import task from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy import log -from scrapy.project import crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender -from scrapy.conf import settings from scrapy.stats import stats from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported from scrapy.utils.engine import get_engine_status class MemoryUsage(object): - def __init__(self): - if not settings.getbool('MEMUSAGE_ENABLED'): + def __init__(self, crawler): + if not crawler.settings.getbool('MEMUSAGE_ENABLED'): raise NotConfigured if not procfs_supported(): raise NotConfigured + self.crawler = crawler self.warned = False - self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY') - self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 - self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 - self.report = settings.getbool('MEMUSAGE_REPORT') + self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY') + self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 + self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 + self.report = crawler.settings.getbool('MEMUSAGE_REPORT') self.mail = MailSender() dispatcher.connect(self.engine_started, signal=signals.engine_started) dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + def get_virtual_size(self): return get_vmvalue_from_procfs('VmSize') @@ -70,10 +73,10 @@ class MemoryUsage(object): log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR) if self.notify_mails: subj = "%s terminated: memory usage exceeded %dM at %s" % \ - (settings['BOT_NAME'], mem, socket.gethostname()) + (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) stats.set_value('memusage/limit_notified', 1) - crawler.stop() + self.crawler.stop() def _check_warning(self): if self.warned: # warn only once @@ -84,7 +87,7 @@ class MemoryUsage(object): log.msg("Memory usage reached %dM" % mem, level=log.WARNING) if self.notify_mails: subj = "%s warning: memory usage reached %dM at %s" % \ - (settings['BOT_NAME'], mem, socket.gethostname()) + (self.crawler.settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) stats.set_value('memusage/warning_notified', 1) self.warned = True diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 3ab7d8659..6fb1d4d78 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -17,12 +17,14 @@ class MediaPipeline(object): self.downloaded = {} self.waiting = defaultdict(list) - def __init__(self, download_func=None): + def __init__(self, download_func=None, crawler=None): self.spiderinfo = {} self.download_func = download_func - if not download_func: - from scrapy.project import crawler - self.crawler = crawler + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler=crawler) def open_spider(self, spider): self.spiderinfo[spider] = self.SpiderInfo(spider) diff --git a/scrapy/contrib/webservice/crawler.py b/scrapy/contrib/webservice/crawler.py index 04c38ce02..f25d90474 100644 --- a/scrapy/contrib/webservice/crawler.py +++ b/scrapy/contrib/webservice/crawler.py @@ -1,10 +1,8 @@ from scrapy.webservice import JsonRpcResource -from scrapy.project import crawler class CrawlerResource(JsonRpcResource): ws_name = 'crawler' - def __init__(self, _crawler=crawler): - JsonRpcResource.__init__(self) - self._target = _crawler + def __init__(self, crawler): + JsonRpcResource.__init__(self, crawler, crawler) diff --git a/scrapy/contrib/webservice/enginestatus.py b/scrapy/contrib/webservice/enginestatus.py index 2e2469c01..20e806423 100644 --- a/scrapy/contrib/webservice/enginestatus.py +++ b/scrapy/contrib/webservice/enginestatus.py @@ -1,19 +1,17 @@ from scrapy.webservice import JsonResource -from scrapy.project import crawler from scrapy.utils.engine import get_engine_status class EngineStatusResource(JsonResource): ws_name = 'enginestatus' - def __init__(self, spider_name=None, _crawler=crawler): - JsonResource.__init__(self) + def __init__(self, crawler, spider_name=None): + JsonResource.__init__(self, crawler) self._spider_name = spider_name self.isLeaf = spider_name is not None - self._crawler = _crawler def render_GET(self, txrequest): - status = get_engine_status(self._crawler.engine) + status = get_engine_status(self.crawler.engine) if self._spider_name is None: return status for sp, st in status['spiders'].items(): @@ -21,4 +19,4 @@ class EngineStatusResource(JsonResource): return st def getChild(self, name, txrequest): - return EngineStatusResource(name, self._crawler) + return EngineStatusResource(name, self.crawler) diff --git a/scrapy/contrib/webservice/stats.py b/scrapy/contrib/webservice/stats.py index 3ab2efce3..623758c20 100644 --- a/scrapy/contrib/webservice/stats.py +++ b/scrapy/contrib/webservice/stats.py @@ -5,6 +5,5 @@ class StatsResource(JsonRpcResource): ws_name = 'stats' - def __init__(self, _stats=stats): - JsonRpcResource.__init__(self) - self._target = _stats + def __init__(self, crawler): + JsonRpcResource.__init__(self, crawler, stats) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index cd60d4d83..b4768f5a9 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -62,15 +62,15 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader(object): - def __init__(self, settings): - self.settings = settings + def __init__(self, crawler): + self.settings = crawler.settings self.slots = {} self.active = set() self.handlers = DownloadHandlers() - self.total_concurrency = settings.getint('CONCURRENT_REQUESTS') - self.domain_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') - self.ip_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_IP') - self.middleware = DownloaderMiddlewareManager.from_settings(settings) + self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS') + self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') + self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP') + self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) def fetch(self, request, spider): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index ab54cb94e..e545e89c2 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -51,15 +51,15 @@ class Slot(object): class ExecutionEngine(object): - def __init__(self, settings, spider_closed_callback): - self.settings = settings + def __init__(self, crawler, spider_closed_callback): + self.settings = crawler.settings self.slots = {} self.running = False self.paused = False - self.scheduler_cls = load_object(settings['SCHEDULER']) - self.downloader = Downloader(self.settings) - self.scraper = Scraper(self, self.settings) - self._concurrent_spiders = settings.getint('CONCURRENT_SPIDERS') + self.scheduler_cls = load_object(self.settings['SCHEDULER']) + self.downloader = Downloader(crawler) + self.scraper = Scraper(crawler) + self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS') self._spider_closed_callback = spider_closed_callback @defer.inlineCallbacks diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 4423134a8..9e46b6b59 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -61,13 +61,13 @@ class Slot(object): class Scraper(object): - def __init__(self, engine, settings): + def __init__(self, crawler): self.slots = {} - self.spidermw = SpiderMiddlewareManager.from_settings(settings) - itemproc_cls = load_object(settings['ITEM_PROCESSOR']) - self.itemproc = itemproc_cls.from_settings(settings) - self.concurrent_items = settings.getint('CONCURRENT_ITEMS') - self.engine = engine + self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) + itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) + self.itemproc = itemproc_cls.from_crawler(crawler) + self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS') + self.crawler = crawler @defer.inlineCallbacks def open_spider(self, spider): @@ -143,7 +143,7 @@ class Scraper(object): def handle_spider_error(self, _failure, request, response, spider): exc = _failure.value if isinstance(exc, CloseSpider): - self.engine.close_spider(spider, exc.reason or 'cancelled') + self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return log.err(_failure, "Spider error processing %s" % request, spider=spider) send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \ @@ -166,7 +166,7 @@ class Scraper(object): if isinstance(output, Request): send_catch_log(signal=signals.request_received, request=output, \ spider=spider) - self.engine.crawl(request=output, spider=spider) + self.crawler.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): self.slots[spider].itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index b5ac99620..4386e7c64 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -31,10 +31,10 @@ class Crawler(object): if self.configured: return self.configured = True - self.extensions = ExtensionManager.from_settings(self.settings) + self.extensions = ExtensionManager.from_crawler(self) spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) self.spiders = spman_cls.from_settings(self.settings) - self.engine = ExecutionEngine(self.settings, self._spider_closed) + self.engine = ExecutionEngine(self, self._spider_closed) def crawl(self, spider, requests=None): spider.set_crawler(self) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index c83b0d44f..031d27846 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -21,13 +21,15 @@ class MiddlewareManager(object): raise NotImplementedError @classmethod - def from_settings(cls, settings): + def from_settings(cls, settings, crawler=None): mwlist = cls._get_mwlist_from_settings(settings) middlewares = [] for clspath in mwlist: try: mwcls = load_object(clspath) - if hasattr(mwcls, 'from_settings'): + if crawler and hasattr(mwcls, 'from_crawler'): + mw = mwcls.from_crawler(crawler) + elif hasattr(mwcls, 'from_settings'): mw = mwcls.from_settings(settings) else: mw = mwcls() @@ -41,6 +43,10 @@ class MiddlewareManager(object): level=log.DEBUG) return cls(*middlewares) + @classmethod + def from_crawler(cls, crawler): + return cls.from_settings(crawler.settings, crawler) + def _add_middleware(self, mw): if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) diff --git a/scrapy/project.py b/scrapy/project.py index bbcb0fd27..bbe947761 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,11 +1,13 @@ """ -This module contains the Scrapy Crawler once installed by calling the crawler -``install`` method, like this:: +--------- WARNING: THIS MODULE IS DEPRECATED ----------- - crawler.install() +This module is deprecated. If you want to get the Scrapy crawler from your +extension, middleware or pipeline implement the `from_crawler` class method. -After that, you can import the (singleton) crawler like this:: +For example: - from scrapy.project import crawler + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) """ diff --git a/scrapy/telnet.py b/scrapy/telnet.py index cd52b0b8d..b6b4c1b06 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -12,14 +12,12 @@ from twisted.internet import protocol from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured -from scrapy.project import crawler from scrapy.stats import stats from scrapy import log, signals from scrapy.utils.signal import send_catch_log from scrapy.utils.trackref import print_live_refs from scrapy.utils.engine import print_engine_status from scrapy.utils.reactor import listen_tcp -from scrapy.conf import settings try: import guppy @@ -34,15 +32,20 @@ update_telnet_vars = object() class TelnetConsole(protocol.ServerFactory): - def __init__(self): - if not settings.getbool('TELNETCONSOLE_ENABLED'): + def __init__(self, crawler): + if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured + self.crawler = crawler self.noisy = False - self.portrange = map(int, settings.getlist('TELNETCONSOLE_PORT')) - self.host = settings['TELNETCONSOLE_HOST'] + self.portrange = map(int, crawler.settings.getlist('TELNETCONSOLE_PORT')) + self.host = crawler.settings['TELNETCONSOLE_HOST'] dispatcher.connect(self.start_listening, signals.engine_started) dispatcher.connect(self.stop_listening, signals.engine_stopped) + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + def start_listening(self): self.port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost() @@ -59,13 +62,13 @@ class TelnetConsole(protocol.ServerFactory): def _get_telnet_vars(self): # Note: if you add entries here also update topics/telnetconsole.rst telnet_vars = { - 'engine': crawler.engine, - 'manager': crawler, - 'extensions': crawler.extensions, + 'engine': self.crawler.engine, + 'manager': self.crawler, + 'extensions': self.crawler.extensions, 'stats': stats, - 'spiders': crawler.spiders, - 'settings': settings, - 'est': print_engine_status, + 'spiders': self.crawler.spiders, + 'settings': self.crawler.settings, + 'est': lambda x: print_engine_status(self.crawler), 'p': pprint.pprint, 'prefs': print_live_refs, 'hpy': hpy, diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 6f4daed73..26fe361ef 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -2,13 +2,8 @@ from time import time # used in global tests code -from scrapy.project import crawler - -def get_engine_status(engine=None): +def get_engine_status(engine): """Return a report of the current engine status""" - if engine is None: - engine = crawler.engine - global_tests = [ "time()-engine.start_time", "engine.has_capacity()", @@ -59,6 +54,6 @@ def format_engine_status(engine=None): s += " %-50s : %s\n" % (test, result) return s -def print_engine_status(engine=None): +def print_engine_status(engine): print format_engine_status(engine) diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index a7faf3899..530c925c4 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -19,9 +19,7 @@ class SpiderReferencer(object): spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$') - def __init__(self, crawler=None): - if crawler is None: - from scrapy.project import crawler + def __init__(self, crawler): self.crawler = crawler def get_reference_from_spider(self, spider): @@ -80,7 +78,8 @@ class ScrapyJSONEncoder(json.JSONEncoder): TIME_FORMAT = "%H:%M:%S" def __init__(self, *a, **kw): - self.spref = kw.pop('spref', None) or SpiderReferencer() + crawler = kw.pop('crawler', None) + self.spref = kw.pop('spref', None) or SpiderReferencer(crawler) super(ScrapyJSONEncoder, self).__init__(*a, **kw) def encode(self, o): diff --git a/scrapy/webservice.py b/scrapy/webservice.py index 64437cc81..58e3f2e0e 100644 --- a/scrapy/webservice.py +++ b/scrapy/webservice.py @@ -15,19 +15,21 @@ from scrapy.utils.misc import load_object from scrapy.utils.txweb import JsonResource as JsonResource_ from scrapy.utils.reactor import listen_tcp from scrapy.utils.conf import build_component_list -from scrapy.conf import settings class JsonResource(JsonResource_): - json_encoder = ScrapyJSONEncoder() + def __init__(self, crawler, target=None): + JsonResource_.__init__(self) + self.crawler = crawler + self.json_encoder = ScrapyJSONEncoder(crawler=crawler) class JsonRpcResource(JsonResource): - json_decoder = ScrapyJSONDecoder() - - def __init__(self, target=None): - JsonResource.__init__(self) + def __init__(self, crawler, target=None): + JsonResource.__init__(self, crawler, target) + self.json_decoder = ScrapyJSONDecoder(crawler=crawler) + self.crawler = crawler self._target = target def render_GET(self, txrequest): @@ -63,23 +65,28 @@ class RootResource(JsonResource): class WebService(server.Site): - def __init__(self): - if not settings.getbool('WEBSERVICE_ENABLED'): + def __init__(self, crawler): + if not crawler.settings.getbool('WEBSERVICE_ENABLED'): raise NotConfigured - logfile = settings['WEBSERVICE_LOGFILE'] - self.portrange = map(int, settings.getlist('WEBSERVICE_PORT')) - self.host = settings['WEBSERVICE_HOST'] - root = RootResource() - reslist = build_component_list(settings['WEBSERVICE_RESOURCES_BASE'], \ - settings['WEBSERVICE_RESOURCES']) + self.crawler = crawler + logfile = crawler.settings['WEBSERVICE_LOGFILE'] + self.portrange = map(int, crawler.settings.getlist('WEBSERVICE_PORT')) + self.host = crawler.settings['WEBSERVICE_HOST'] + root = RootResource(crawler) + reslist = build_component_list(crawler.settings['WEBSERVICE_RESOURCES_BASE'], \ + crawler.settings['WEBSERVICE_RESOURCES']) for res_cls in map(load_object, reslist): - res = res_cls() + res = res_cls(crawler) root.putChild(res.ws_name, res) server.Site.__init__(self, root, logPath=logfile) self.noisy = False dispatcher.connect(self.start_listening, signals.engine_started) dispatcher.connect(self.stop_listening, signals.engine_stopped) + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + def start_listening(self): self.port = listen_tcp(self.portrange, self.host, self) h = self.port.getHost()