diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index db2c0b3cd..66c89349c 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -10,7 +10,7 @@ from twisted.internet import reactor from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.conf import settings class CloseSpider(object): @@ -29,13 +29,14 @@ class CloseSpider(object): dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def spider_opened(self, spider): - self.tasks[spider] = reactor.callLater(self.timeout, scrapyengine.close_spider, \ - spider=spider, reason='closespider_timeout') + self.tasks[spider] = reactor.callLater(self.timeout, \ + scrapymanager.engine.close_spider, spider=spider, \ + reason='closespider_timeout') def item_passed(self, item, spider): self.counts[spider] += 1 if self.counts[spider] == self.itempassed: - scrapyengine.close_spider(spider, 'closespider_itempassed') + scrapymanager.engine.close_spider(spider, 'closespider_itempassed') def spider_closed(self, spider): self.counts.pop(spider, None) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 3784ff9d2..1f877708c 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -9,7 +9,7 @@ import robotparser from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured, IgnoreRequest from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -42,7 +42,7 @@ class RobotsTxtMiddleware(object): self._parsers[netloc] = None robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc) robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY) - dfd = scrapyengine.download(robotsreq, spider) + dfd = scrapymanager.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots) self._spider_netlocs[spider].add(netloc) return self._parsers[netloc] diff --git a/scrapy/contrib/itemsampler.py b/scrapy/contrib/itemsampler.py index 9f8666467..9098f09e1 100644 --- a/scrapy/contrib/itemsampler.py +++ b/scrapy/contrib/itemsampler.py @@ -27,7 +27,7 @@ import cPickle as pickle from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured from scrapy.core import signals from scrapy.utils.response import response_httprepr @@ -60,7 +60,7 @@ class ItemSamplerPipeline(object): stats.set_value("items_sampled", sampled, spider=spider) log.msg("Sampled %s" % item, spider=spider, level=log.INFO) if close_spider and sampled == items_per_spider: - scrapyengine.close_spider(spider) + scrapymanager.engine.close_spider(spider) return item def engine_stopped(self): diff --git a/scrapy/contrib/memusage.py b/scrapy/contrib/memusage.py index cb53e44e5..0274753f2 100644 --- a/scrapy/contrib/memusage.py +++ b/scrapy/contrib/memusage.py @@ -13,7 +13,6 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals from scrapy import log from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine from scrapy.core.exceptions import NotConfigured from scrapy.mail import MailSender from scrapy.conf import settings diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 96c804be3..10debf84d 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -4,7 +4,7 @@ from twisted.internet.defer import Deferred, DeferredList from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy import log from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -103,7 +103,7 @@ class MediaPipeline(object): """ request.priority = self.DOWNLOAD_PRIORITY - return scrapyengine.download(request, info.spider) + return scrapymanager.engine.download(request, info.spider) def media_to_download(self, request, info): """ Ongoing request hook pre-cache diff --git a/scrapy/contrib/spiderclosedelay.py b/scrapy/contrib/spiderclosedelay.py index 3ecee8dfb..f8e92cfee 100644 --- a/scrapy/contrib/spiderclosedelay.py +++ b/scrapy/contrib/spiderclosedelay.py @@ -9,7 +9,7 @@ from scrapy.xlib.pydispatch import dispatcher from collections import defaultdict from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured, DontCloseSpider from scrapy.conf import settings @@ -26,7 +26,7 @@ class SpiderCloseDelay(object): def spider_idle(self, spider): try: - lastseen = scrapyengine.downloader.sites[spider].lastseen + lastseen = scrapymanager.engine.downloader.sites[spider].lastseen except KeyError: lastseen = None if not lastseen: diff --git a/scrapy/contrib/spidermiddleware/requestlimit.py b/scrapy/contrib/spidermiddleware/requestlimit.py index 3d8516801..04138588d 100644 --- a/scrapy/contrib/spidermiddleware/requestlimit.py +++ b/scrapy/contrib/spidermiddleware/requestlimit.py @@ -7,7 +7,7 @@ from itertools import imap from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.core.exceptions import NotConfigured from scrapy.conf import settings from scrapy.http import Request @@ -61,5 +61,5 @@ class RequestLimitMiddleware(object): return request_or_other def _pending_count(self, spider): - pending = scrapyengine.scheduler.pending_requests.get(spider, []) + pending = scrapymanager.engine.scheduler.pending_requests.get(spider, []) return len(pending) diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 2e61e8bc3..c5badee0e 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -4,7 +4,7 @@ Live statistics extension from datetime import datetime from scrapy.xlib.pydispatch import dispatcher from scrapy.core import signals -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.management.web import banner, webconsole_discover_module class SpiderStats(object): @@ -44,8 +44,8 @@ class LiveStats(object): self.domains[spider].crawled += 1 def webconsole_render(self, wc_request): - sch = scrapyengine.scheduler - dwl = scrapyengine.downloader + sch = scrapymanager.engine.scheduler + dwl = scrapymanager.engine.downloader totdomains = totscraped = totcrawled = totscheduled = totactive = totdqueued = tottransf = 0 s = banner(self) diff --git a/scrapy/contrib/webconsole/scheduler.py b/scrapy/contrib/webconsole/scheduler.py index d94db13ac..756bc3b6d 100644 --- a/scrapy/contrib/webconsole/scheduler.py +++ b/scrapy/contrib/webconsole/scheduler.py @@ -5,7 +5,7 @@ See documentation in docs/topics/extensions.rst """ from scrapy.xlib.pydispatch import dispatcher -from scrapy.core.engine import scrapyengine +from scrapy.core.manager import scrapymanager from scrapy.management.web import banner, webconsole_discover_module class SchedulerQueue(object): @@ -21,7 +21,7 @@ class SchedulerQueue(object): def webconsole_render(self, wc_request): s = banner(self) s += "