From faf7a7da835a4c43294f92b67c37d4b899bbd905 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Sun, 22 Aug 2010 02:10:53 -0300 Subject: [PATCH] Moved scrapymanager singleton to scrapy.project module. Refs #189 Detail of changes: * Moved scrapy.core.manager.ExecutionManager class to scrapy.crawler.Crawler * Added scrapy.project.crawler singleton to reference a singleton instance of Crawler class (previously known as scrapymanager) * Left an alias scrapy.core.manager.scrapymanager to scrapy.project.crawler for backwards compatibility (to be removed in Scrapy 0.11) --- docs/topics/telnetconsole.rst | 2 +- scrapy/cmdline.py | 4 +- scrapy/commands/crawl.py | 6 +- scrapy/commands/fetch.py | 8 +- scrapy/commands/parse.py | 8 +- scrapy/commands/runserver.py | 6 +- scrapy/commands/runspider.py | 6 +- scrapy/commands/shell.py | 10 +- scrapy/contrib/closespider.py | 6 +- .../contrib/downloadermiddleware/robotstxt.py | 4 +- scrapy/contrib/memusage.py | 4 +- scrapy/contrib/pipeline/media.py | 4 +- scrapy/contrib/spiderclosedelay.py | 4 +- .../contrib/spidermiddleware/requestlimit.py | 4 +- scrapy/contrib/webservice/enginestatus.py | 4 +- scrapy/contrib/webservice/manager.py | 4 +- scrapy/core/engine.py | 3 +- scrapy/core/manager.py | 102 +----------------- scrapy/crawler.py | 96 +++++++++++++++++ scrapy/project.py | 3 + scrapy/shell.py | 4 +- scrapy/telnet.py | 6 +- scrapy/tests/test_engine.py | 8 +- scrapy/utils/engine.py | 6 +- scrapy/utils/serialize.py | 4 +- 25 files changed, 161 insertions(+), 155 deletions(-) create mode 100644 scrapy/crawler.py create mode 100644 scrapy/project.py diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 99b6e627e..770b2ba53 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -45,7 +45,7 @@ convenience: +================+===================================================================+ | ``engine`` | the Scrapy engine object (``scrapy.core.engine.scrapyengine``) | +----------------+-------------------------------------------------------------------+ -| ``manager`` | the Scrapy manager object (``scrapy.core.manager.scrapymanager``) | +| ``manager`` | the Project Crawler object (``scrapy.project.crawler``) | +----------------+-------------------------------------------------------------------+ | ``extensions`` | the extensions object (``scrapy.extension.extensions``) | +----------------+-------------------------------------------------------------------+ diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 9ec7e6ab0..8d437a1c5 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -113,8 +113,8 @@ def execute(argv=None): settings.defaults.update(cmd.default_settings) del args[0] # remove command name from args - from scrapy.core.manager import scrapymanager - scrapymanager.configure(control_reactor=True) + from scrapy.project import crawler + crawler.configure(control_reactor=True) ret = _run_command(cmd, args, opts) if ret is False: parser.print_help() diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 63b8054bc..28a032bb3 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,7 +1,7 @@ from scrapy import log from scrapy.command import ScrapyCommand from scrapy.core.queue import ExecutionQueue -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.conf import settings from scrapy.http import Request from scrapy.spider import spiders @@ -50,8 +50,8 @@ class Command(ScrapyCommand): for url in urls: q.append_url(url, spider) - scrapymanager.queue = q - scrapymanager.start() + crawler.queue = q + crawler.start() def _group_urls_by_spider(self, urls): spider_urls = defaultdict(list) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 3ed3674da..766d7cb5f 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -2,7 +2,7 @@ import pprint from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.http import Request from scrapy.spider import BaseSpider, spiders from scrapy.utils.url import is_url @@ -47,8 +47,8 @@ class Command(ScrapyCommand): except KeyError: log.msg("Could not find spider: %s" % opts.spider, log.ERROR) - scrapymanager.configure() - scrapymanager.queue.append_request(request, spider, \ + crawler.configure() + crawler.queue.append_request(request, spider, \ default_spider=BaseSpider('default')) - scrapymanager.start() + crawler.start() diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 2d548c39e..b63129d67 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,5 +1,5 @@ from scrapy.command import ScrapyCommand -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.http import Request from scrapy.item import BaseItem from scrapy.spider import spiders @@ -88,9 +88,9 @@ class Command(ScrapyCommand): log.msg('Unable to find spider for URL: %s' % args[0], log.ERROR) return - scrapymanager.configure() - scrapymanager.queue.append_request(request, spider) - scrapymanager.start() + crawler.configure() + crawler.queue.append_request(request, spider) + crawler.start() if not responses: log.msg('No response returned', log.ERROR, spider=spider) diff --git a/scrapy/commands/runserver.py b/scrapy/commands/runserver.py index c6371fdf4..9cd6778b1 100644 --- a/scrapy/commands/runserver.py +++ b/scrapy/commands/runserver.py @@ -1,5 +1,5 @@ from scrapy.command import ScrapyCommand -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.utils.misc import load_object from scrapy.conf import settings @@ -12,5 +12,5 @@ class Command(ScrapyCommand): def run(self, args, opts): queue_class = load_object(settings['SERVICE_QUEUE']) - scrapymanager.queue = queue_class() - scrapymanager.start() + crawler.queue = queue_class() + crawler.start() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index f9cb7a1c4..98d80ddbb 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,7 @@ import os from scrapy import log from scrapy.utils.spider import iter_spider_classes from scrapy.command import ScrapyCommand -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler def _import_file(filepath): abspath = os.path.abspath(filepath) @@ -52,5 +52,5 @@ class Command(ScrapyCommand): return spider = spclasses.pop()() # schedule spider and start engine - scrapymanager.queue.append_spider(spider) - scrapymanager.start() + crawler.queue.append_spider(spider) + crawler.start() diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 463bda238..2d0e04d45 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,7 +4,7 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.core.queue import KeepAliveExecutionQueue from scrapy.command import ScrapyCommand from scrapy.shell import Shell @@ -30,7 +30,7 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None - shell = Shell(scrapymanager, update_vars=self.update_vars, inthread=True) - shell.start(url=url).addBoth(lambda _: scrapymanager.stop()) - scrapymanager.queue = KeepAliveExecutionQueue() - scrapymanager.start() + shell = Shell(crawler, update_vars=self.update_vars, inthread=True) + shell.start(url=url).addBoth(lambda _: crawler.stop()) + crawler.queue = KeepAliveExecutionQueue() + crawler.start() diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index b54554b9d..a334c155a 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -10,7 +10,7 @@ from twisted.internet import reactor from scrapy.xlib.pydispatch import dispatcher from scrapy import signals -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.conf import settings class CloseSpider(object): @@ -30,13 +30,13 @@ class CloseSpider(object): def spider_opened(self, spider): self.tasks[spider] = reactor.callLater(self.timeout, \ - scrapymanager.engine.close_spider, spider=spider, \ + crawler.engine.close_spider, spider=spider, \ reason='closespider_timeout') def item_passed(self, item, spider): self.counts[spider] += 1 if self.counts[spider] == self.itempassed: - scrapymanager.engine.close_spider(spider, 'closespider_itempassed') + crawler.engine.close_spider(spider, 'closespider_itempassed') def spider_closed(self, spider): self.counts.pop(spider, None) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index d359f7ac1..fe47317f7 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -9,7 +9,7 @@ import robotparser from scrapy.xlib.pydispatch import dispatcher from scrapy import signals -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -42,7 +42,7 @@ class RobotsTxtMiddleware(object): self._parsers[netloc] = None robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY) - dfd = scrapymanager.engine.download(robotsreq, spider) + dfd = crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots) self._spider_netlocs[spider].add(netloc) return self._parsers[netloc] diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index 68e6763a2..3860ef52d 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -13,7 +13,7 @@ from twisted.internet import task from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy import log -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.conf import settings @@ -74,7 +74,7 @@ class MemoryUsage(object): (settings['BOT_NAME'], mem, socket.gethostname()) self._send_report(self.notify_mails, subj) stats.set_value('memusage/limit_notified', 1) - scrapymanager.stop() + crawler.stop() def _check_warning(self): if self.warned: # warn only once diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 5a302d8c1..752a3c144 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -2,7 +2,7 @@ from twisted.internet.defer import Deferred, DeferredList from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy import log -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -87,7 +87,7 @@ class MediaPipeline(object): def download(self, request, info): """Defines how to download the media request""" request.priority = self.DOWNLOAD_PRIORITY - return scrapymanager.engine.download(request, info.spider) + return crawler.engine.download(request, info.spider) def media_to_download(self, request, info): """Check request before starting download""" diff --git a/scrapy/contrib/spiderclosedelay.py b/scrapy/contrib/spiderclosedelay.py index 458a8dcd5..203c4de62 100644 --- a/scrapy/contrib/spiderclosedelay.py +++ b/scrapy/contrib/spiderclosedelay.py @@ -9,7 +9,7 @@ from scrapy.xlib.pydispatch import dispatcher from collections import defaultdict from scrapy import signals -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.exceptions import NotConfigured, DontCloseSpider from scrapy.conf import settings @@ -26,7 +26,7 @@ class SpiderCloseDelay(object): def spider_idle(self, spider): try: - lastseen = scrapymanager.engine.downloader.sites[spider].lastseen + lastseen = crawler.engine.downloader.sites[spider].lastseen except KeyError: lastseen = None if not lastseen: diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py index 233b87d7e..bd52576f7 100644 --- a/scrapy/contrib/spidermiddleware/requestlimit.py +++ b/scrapy/contrib/spidermiddleware/requestlimit.py @@ -7,7 +7,7 @@ from itertools import imap from scrapy.xlib.pydispatch import dispatcher from scrapy import signals -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.exceptions import NotConfigured from scrapy.conf import settings from scrapy.http import Request @@ -61,5 +61,5 @@ class RequestLimitMiddleware(object): return request_or_other def _pending_count(self, spider): - pending = scrapymanager.engine.scheduler.pending_requests.get(spider, []) + pending = crawler.engine.scheduler.pending_requests.get(spider, []) return len(pending) diff --git a/scrapy/contrib/webservice/enginestatus.py b/scrapy/contrib/webservice/enginestatus.py index 66230fab0..f7117de67 100644 --- a/scrapy/contrib/webservice/enginestatus.py +++ b/scrapy/contrib/webservice/enginestatus.py @@ -1,12 +1,12 @@ from scrapy.webservice import JsonResource -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.utils.engine import get_engine_status class EngineStatusResource(JsonResource): ws_name = 'enginestatus' - def __init__(self, spider_name=None, _manager=scrapymanager): + def __init__(self, spider_name=None, _manager=crawler): JsonResource.__init__(self) self._spider_name = spider_name self.isLeaf = spider_name is not None diff --git a/scrapy/contrib/webservice/manager.py b/scrapy/contrib/webservice/manager.py index 87bc4e270..520f9f808 100644 --- a/scrapy/contrib/webservice/manager.py +++ b/scrapy/contrib/webservice/manager.py @@ -1,10 +1,10 @@ from scrapy.webservice import JsonRpcResource -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler class ManagerResource(JsonRpcResource): ws_name = 'manager' - def __init__(self, _manager=scrapymanager): + def __init__(self, _manager=crawler): JsonRpcResource.__init__(self) self._target = _manager diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 38c3c9526..082cb9483 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -24,7 +24,8 @@ from scrapy.utils.defer import mustbe_deferred class ExecutionEngine(object): - def __init__(self): + def __init__(self, crawler): + self.crawler = crawler self.configured = False self.closing = {} # dict (spider -> reason) of spiders being closed self.closing_dfds = {} # dict (spider -> deferred) of spiders being closed diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index a26a25144..59e51495c 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -1,98 +1,6 @@ -import signal +import warnings +warnings.warn("scrapy.core.manager.scrapymanager is deprecated and will be removed in Scrapy 0.11, use scrapy.project.crawler instead", \ + DeprecationWarning, stacklevel=2) -from twisted.internet import reactor, defer - -from scrapy.core.engine import ExecutionEngine -from scrapy.core.queue import ExecutionQueue -from scrapy.extension import extensions -from scrapy import log -from scrapy.spider import spiders -from scrapy.utils.ossignal import install_shutdown_handlers, signal_names - - -class ExecutionManager(object): - - def __init__(self): - self.configured = False - self.control_reactor = True - self.engine = ExecutionEngine() - - def configure(self, control_reactor=True, queue=None): - self.control_reactor = control_reactor - if control_reactor: - install_shutdown_handlers(self._signal_shutdown) - - if not log.started: - log.start() - if not extensions.loaded: - extensions.load() - if not spiders.loaded: - spiders.load() - log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()), - level=log.DEBUG) - - self.queue = queue or ExecutionQueue() - self.engine.configure(self._spider_closed) - self.configured = True - - @defer.inlineCallbacks - def _start_next_spider(self): - spider, requests = yield defer.maybeDeferred(self.queue.get_next) - if spider: - self._start_spider(spider, requests) - if self.engine.has_capacity() and not self._nextcall.active(): - self._nextcall = reactor.callLater(self.queue.polling_delay, \ - self._spider_closed) - - @defer.inlineCallbacks - def _start_spider(self, spider, requests): - """Don't call this method. Use self.queue to start new spiders""" - yield defer.maybeDeferred(self.engine.open_spider, spider) - for request in requests: - self.engine.crawl(request, spider) - - @defer.inlineCallbacks - def _spider_closed(self, spider=None): - if not self.engine.open_spiders: - is_finished = yield defer.maybeDeferred(self.queue.is_finished) - if is_finished: - self.stop() - return - if self.engine.has_capacity(): - self._start_next_spider() - - @defer.inlineCallbacks - def start(self): - yield defer.maybeDeferred(self.engine.start) - self._nextcall = reactor.callLater(0, self._start_next_spider) - reactor.addSystemEventTrigger('before', 'shutdown', self.stop) - if self.control_reactor: - reactor.run(installSignalHandlers=False) - - @defer.inlineCallbacks - def stop(self): - if self._nextcall.active(): - self._nextcall.cancel() - if self.engine.running: - yield defer.maybeDeferred(self.engine.stop) - try: - reactor.stop() - except RuntimeError: # raised if already stopped or in shutdown stage - pass - - def _signal_shutdown(self, signum, _): - signame = signal_names[signum] - log.msg("Received %s, shutting down gracefully. Send again to force " \ - "unclean shutdown" % signame, level=log.INFO) - reactor.callFromThread(self.stop) - install_shutdown_handlers(self._signal_kill) - - def _signal_kill(self, signum, _): - signame = signal_names[signum] - log.msg('Received %s twice, forcing unclean shutdown' % signame, \ - level=log.INFO) - log.log_level = log.SILENT # disable logging of confusing tracebacks - reactor.callFromThread(self.engine.kill) - install_shutdown_handlers(signal.SIG_IGN) - -scrapymanager = ExecutionManager() +from scrapy.project import crawler +scrapymanager = crawler diff --git a/scrapy/crawler.py b/scrapy/crawler.py new file mode 100644 index 000000000..fcf56a79d --- /dev/null +++ b/scrapy/crawler.py @@ -0,0 +1,96 @@ +import signal + +from twisted.internet import reactor, defer + +from scrapy.core.engine import ExecutionEngine +from scrapy.core.queue import ExecutionQueue +from scrapy.extension import extensions +from scrapy import log +from scrapy.spider import spiders +from scrapy.utils.ossignal import install_shutdown_handlers, signal_names + + +class Crawler(object): + + def __init__(self): + self.configured = False + self.control_reactor = True + self.engine = ExecutionEngine(self) + + def configure(self, control_reactor=True, queue=None): + self.control_reactor = control_reactor + if control_reactor: + install_shutdown_handlers(self._signal_shutdown) + + if not log.started: + log.start() + if not extensions.loaded: + extensions.load() + if not spiders.loaded: + spiders.load() + log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()), + level=log.DEBUG) + + self.queue = queue or ExecutionQueue() + self.engine.configure(self._spider_closed) + self.configured = True + + @defer.inlineCallbacks + def _start_next_spider(self): + spider, requests = yield defer.maybeDeferred(self.queue.get_next) + if spider: + self._start_spider(spider, requests) + if self.engine.has_capacity() and not self._nextcall.active(): + self._nextcall = reactor.callLater(self.queue.polling_delay, \ + self._spider_closed) + + @defer.inlineCallbacks + def _start_spider(self, spider, requests): + """Don't call this method. Use self.queue to start new spiders""" + yield defer.maybeDeferred(self.engine.open_spider, spider) + for request in requests: + self.engine.crawl(request, spider) + + @defer.inlineCallbacks + def _spider_closed(self, spider=None): + if not self.engine.open_spiders: + is_finished = yield defer.maybeDeferred(self.queue.is_finished) + if is_finished: + self.stop() + return + if self.engine.has_capacity(): + self._start_next_spider() + + @defer.inlineCallbacks + def start(self): + yield defer.maybeDeferred(self.engine.start) + self._nextcall = reactor.callLater(0, self._start_next_spider) + reactor.addSystemEventTrigger('before', 'shutdown', self.stop) + if self.control_reactor: + reactor.run(installSignalHandlers=False) + + @defer.inlineCallbacks + def stop(self): + if self._nextcall.active(): + self._nextcall.cancel() + if self.engine.running: + yield defer.maybeDeferred(self.engine.stop) + try: + reactor.stop() + except RuntimeError: # raised if already stopped or in shutdown stage + pass + + def _signal_shutdown(self, signum, _): + signame = signal_names[signum] + log.msg("Received %s, shutting down gracefully. Send again to force " \ + "unclean shutdown" % signame, level=log.INFO) + reactor.callFromThread(self.stop) + install_shutdown_handlers(self._signal_kill) + + def _signal_kill(self, signum, _): + signame = signal_names[signum] + log.msg('Received %s twice, forcing unclean shutdown' % signame, \ + level=log.INFO) + log.log_level = log.SILENT # disable logging of confusing tracebacks + reactor.callFromThread(self.engine.kill) + install_shutdown_handlers(signal.SIG_IGN) diff --git a/scrapy/project.py b/scrapy/project.py new file mode 100644 index 000000000..cb2ccd26c --- /dev/null +++ b/scrapy/project.py @@ -0,0 +1,3 @@ +from scrapy.crawler import Crawler + +crawler = Crawler() diff --git a/scrapy/shell.py b/scrapy/shell.py index af6060cd8..693f0f6e3 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -107,5 +107,5 @@ class Shell(object): def inspect_response(response, spider=None): """Open a shell to inspect the given response""" - from scrapy.core.manager import scrapymanager - Shell(scrapymanager).start(response=response, spider=spider) + from scrapy.project import crawler + Shell(crawler).start(response=response, spider=spider) diff --git a/scrapy/telnet.py b/scrapy/telnet.py index fa46becfa..4a57888ab 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -12,7 +12,7 @@ from twisted.internet import reactor, protocol from scrapy.extension import extensions from scrapy.exceptions import NotConfigured -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.spider import spiders from scrapy.stats import stats from scrapy.utils.signal import send_catch_log @@ -48,8 +48,8 @@ class TelnetConsole(protocol.ServerFactory): def _get_telnet_vars(self): # Note: if you add entries here also update topics/telnetconsole.rst telnet_vars = { - 'engine': scrapymanager.engine, - 'manager': scrapymanager, + 'engine': crawler.engine, + 'manager': crawler, 'extensions': extensions, 'stats': stats, 'spiders': spiders, diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 69c3127fe..79de27fe3 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -8,7 +8,7 @@ from twisted.internet import reactor from twisted.web import server, resource, static, util from scrapy import signals -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.xlib.pydispatch import dispatcher from scrapy.tests import tests_datadir from scrapy.spider import BaseSpider @@ -96,9 +96,9 @@ class CrawlingSession(object): dispatcher.connect(self.request_received, signals.request_received) dispatcher.connect(self.response_downloaded, signals.response_downloaded) - scrapymanager.configure() - scrapymanager.queue.append_spider(self.spider) - scrapymanager.start() + crawler.configure() + crawler.queue.append_spider(self.spider) + crawler.start() self.port.stopListening() self.wasrun = True # FIXME: extremly ugly hack to avoid propagating errors to other diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index a87741631..52d213109 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,13 +1,11 @@ """Some debugging functions for working with the Scrapy engine""" -from time import time - -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler def get_engine_status(engine=None): """Return a report of the current engine status""" if engine is None: - engine = scrapymanager.engine + engine = crawler.engine global_tests = [ "time()-engine.start_time", diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index b0bec94b7..b38963265 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -4,7 +4,7 @@ import decimal from twisted.internet import defer -from scrapy.core.manager import scrapymanager +from scrapy.project import crawler from scrapy.spider import BaseSpider from scrapy.http import Request, Response from scrapy.utils.py26 import json @@ -21,7 +21,7 @@ class SpiderReferencer(object): spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$') def __init__(self, manager=None): - self.manager = manager or scrapymanager + self.manager = manager or crawler def get_reference_from_spider(self, spider): return 'spider:%x:%s' % (id(spider), spider.name)