diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index db2c0b3cd..66c89349c 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -10,7 +10,7 @@ from twisted.internet import reactor from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.conf import settings class CloseSpider(object): @@ -29,13 +29,14 @@ class CloseSpider(object): dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def spider_opened(self, spider): - self.tasks[spider] = reactor.callLater(self.timeout, scrapyengine.close_spider, \ - spider=spider, reason='closespider_timeout') + self.tasks[spider] = reactor.callLater(self.timeout, \ + scrapymanager.engine.close_spider, spider=spider, \ + reason='closespider_timeout') def item_passed(self, item, spider): self.counts[spider] += 1 if self.counts[spider] == self.itempassed: - scrapyengine.close_spider(spider, 'closespider_itempassed') + scrapymanager.engine.close_spider(spider, 'closespider_itempassed') def spider_closed(self, spider): self.counts.pop(spider, None) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 3784ff9d2..1f877708c 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -9,7 +9,7 @@ import robotparser from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -42,7 +42,7 @@ class RobotsTxtMiddleware(object): self._parsers[netloc] = None robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY) - dfd = scrapyengine.download(robotsreq, spider) + dfd = scrapymanager.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots) self._spider_netlocs[spider].add(netloc) return self._parsers[netloc] diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index 9f8666467..9098f09e1 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -27,7 +27,7 @@ import cPickle as pickle from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured from scrapy.core import signals from scrapy.utils.response import response_httprepr @@ -60,7 +60,7 @@ class ItemSamplerPipeline(object): stats.set_value("items_sampled", sampled, spider=spider) log.msg("Sampled %s" % item, spider=spider, level=log.INFO) if close_spider and sampled == items_per_spider: - scrapyengine.close_spider(spider) + scrapymanager.engine.close_spider(spider) return item def engine_stopped(self): diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index cb53e44e5..0274753f2 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -13,7 +13,6 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.conf import settings diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 96c804be3..10debf84d 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -4,7 +4,7 @@ from twisted.internet.defer import Deferred, DeferredList from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy import log from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -103,7 +103,7 @@ class MediaPipeline(object): """ request.priority = self.DOWNLOAD_PRIORITY - return scrapyengine.download(request, info.spider) + return scrapymanager.engine.download(request, info.spider) def media_to_download(self, request, info): """ Ongoing request hook pre-cache diff --git a/scrapy/contrib/spiderclosedelay.py b/scrapy/contrib/spiderclosedelay.py index 3ecee8dfb..f8e92cfee 100644 --- a/scrapy/contrib/spiderclosedelay.py +++ b/scrapy/contrib/spiderclosedelay.py @@ -9,7 +9,7 @@ from scrapy.xlib.pydispatch import dispatcher from collections import defaultdict from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured, DontCloseSpider from scrapy.conf import settings @@ -26,7 +26,7 @@ class SpiderCloseDelay(object): def spider_idle(self, spider): try: - lastseen = scrapyengine.downloader.sites[spider].lastseen + lastseen = scrapymanager.engine.downloader.sites[spider].lastseen except KeyError: lastseen = None if not lastseen: diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py index 3d8516801..04138588d 100644 --- a/scrapy/contrib/spidermiddleware/requestlimit.py +++ b/scrapy/contrib/spidermiddleware/requestlimit.py @@ -7,7 +7,7 @@ from itertools import imap from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured from scrapy.conf import settings from scrapy.http import Request @@ -61,5 +61,5 @@ class RequestLimitMiddleware(object): return request_or_other def _pending_count(self, spider): - pending = scrapyengine.scheduler.pending_requests.get(spider, []) + pending = scrapymanager.engine.scheduler.pending_requests.get(spider, []) return len(pending) diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 2e61e8bc3..c5badee0e 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -4,7 +4,7 @@ Live statistics extension from datetime import datetime from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.management.web import banner, webconsole_discover_module class SpiderStats(object): @@ -44,8 +44,8 @@ class LiveStats(object): self.domains[spider].crawled += 1 def webconsole_render(self, wc_request): - sch = scrapyengine.scheduler - dwl = scrapyengine.downloader + sch = scrapymanager.engine.scheduler + dwl = scrapymanager.engine.downloader totdomains = totscraped = totcrawled = totscheduled = totactive = totdqueued = tottransf = 0 s = banner(self) diff --git a/scrapy/contrib/webconsole/scheduler.py b/scrapy/contrib/webconsole/scheduler.py index d94db13ac..756bc3b6d 100644 --- a/scrapy/contrib/webconsole/scheduler.py +++ b/scrapy/contrib/webconsole/scheduler.py @@ -5,7 +5,7 @@ See documentation in docs/topics/extensions.rst """ from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.management.web import banner, webconsole_discover_module class SchedulerQueue(object): @@ -21,7 +21,7 @@ class SchedulerQueue(object): def webconsole_render(self, wc_request): s = banner(self) s += "
" s += "Scheduled spiders:
" diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 8bc90ccbe..11ce4b35a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -286,5 +286,3 @@ class ExecutionEngine(object): def _finish_stopping_engine(self): send_catch_log(signal=signals.engine_stopped, sender=self.__class__) - -scrapyengine = ExecutionEngine() diff --git a/scrapy/core/manager.py b/scrapy/core/manager.py index d297ed0af..7bac4a3e8 100644 --- a/scrapy/core/manager.py +++ b/scrapy/core/manager.py @@ -2,7 +2,7 @@ import signal from twisted.internet import reactor, defer -from scrapy.core.engine import scrapyengine +from scrapy.core.engine import ExecutionEngine from scrapy.core.queue import ExecutionQueue from scrapy.extension import extensions from scrapy import log @@ -15,7 +15,7 @@ class ExecutionManager(object): def __init__(self): self.configured = False self.control_reactor = True - self.engine = scrapyengine + self.engine = ExecutionEngine() def configure(self, control_reactor=True, queue=None): self.control_reactor = control_reactor diff --git a/scrapy/management/telnet.py b/scrapy/management/telnet.py index 9d0a234ea..ba1315bb8 100644 --- a/scrapy/management/telnet.py +++ b/scrapy/management/telnet.py @@ -13,7 +13,6 @@ from twisted.internet import reactor, protocol from scrapy.extension import extensions from scrapy.core.exceptions import NotConfigured from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine from scrapy.spider import spiders from scrapy.stats import stats from scrapy.utils.trackref import print_live_refs @@ -28,7 +27,7 @@ except ImportError: # if you add entries here also update topics/telnetconsole.rst telnet_namespace = { - 'engine': scrapyengine, + 'engine': scrapymanager.engine, 'manager': scrapymanager, 'extensions': extensions, 'stats': stats, diff --git a/scrapy/management/web.py b/scrapy/management/web.py index e2036bf49..498613a5c 100644 --- a/scrapy/management/web.py +++ b/scrapy/management/web.py @@ -13,7 +13,7 @@ from twisted.web import server, resource from scrapy.xlib.pydispatch import dispatcher from scrapy.core.exceptions import NotConfigured -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.conf import settings # web management signals @@ -38,7 +38,7 @@ def banner(module=None): s += "
Bot: %s | Host: %s | Uptime: %ds
\n" % \ (settings['BOT_NAME'], socket.gethostname(), uptime) if module: diff --git a/scrapy/shell.py b/scrapy/shell.py index 97af70869..4d75e8072 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -17,7 +17,6 @@ from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.conf import settings from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine from scrapy.core.queue import KeepAliveExecutionQueue from scrapy.http import Request, TextResponse @@ -59,8 +58,8 @@ class Shell(object): log_multiple=True) print "Fetching %s..." % request - scrapyengine.open_spider(spider) - response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ + scrapymanager.engine.open_spider(spider) + response = threads.blockingCallFromThread(reactor, scrapymanager.engine.schedule, \ request, spider) if response: self.populate_vars(url, response, request, spider) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index e573730ab..6f37055a5 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -2,12 +2,12 @@ from time import time -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager def get_engine_status(engine=None): """Return a report of the current engine status""" if engine is None: - engine = scrapyengine + engine = scrapymanager.engine global_tests = [ "time()-engine.start_time",