RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Another step towards singleton removal: deprecated crawler singleton import (from scrapy.project import crawler) by a new class method that extensions can implement to receive the crawler

This commit is contained in:
Pablo Hoffman 2011-08-08 11:42:44 -03:00
parent 0eaa1d95f6
commit 5c63b2307f
17 changed files with 125 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scrapy/contrib/closespider.py
View File

@ -13,12 +13,12 @@ from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals, log
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.project import crawler
from scrapy.conf import settings
class CloseSpider(object):
def __init__(self):
def __init__(self, crawler):
self.crawler = crawler
self.timeout = settings.getint('CLOSESPIDER_TIMEOUT')
self.itemcount = settings.getint('CLOSESPIDER_ITEMCOUNT')
# XXX: legacy support - remove for future releases
@ -43,28 +43,32 @@ class CloseSpider(object):
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def catch_log(self, event):
if event.get('logLevel') == log.ERROR:
spider = event.get('spider')
if spider:
self.errorcounts[spider] += 1
if self.errorcounts[spider] == self.errorcount:
crawler.engine.close_spider(spider, 'closespider_errorcount')
self.crawler.engine.close_spider(spider, 'closespider_errorcount')
def page_count(self, response, request, spider):
self.pagecounts[spider] += 1
if self.pagecounts[spider] == self.pagecount:
crawler.engine.close_spider(spider, 'closespider_pagecount')
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
def spider_opened(self, spider):
self.tasks[spider] = reactor.callLater(self.timeout, \
crawler.engine.close_spider, spider=spider, \
self.crawler.engine.close_spider, spider=spider, \
reason='closespider_timeout')
def item_scraped(self, item, spider):
self.counts[spider] += 1
if self.counts[spider] == self.itemcount:
crawler.engine.close_spider(spider, 'closespider_itemcount')
self.crawler.engine.close_spider(spider, 'closespider_itemcount')
def spider_closed(self, spider):
self.counts.pop(spider, None)

13
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -9,25 +9,28 @@ import robotparser
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals, log
from scrapy.project import crawler
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
from scrapy.conf import settings
class RobotsTxtMiddleware(object):
DOWNLOAD_PRIORITY = 1000
def __init__(self):
if not settings.getbool('ROBOTSTXT_OBEY'):
def __init__(self, crawler):
if not crawler.settings.getbool('ROBOTSTXT_OBEY'):
raise NotConfigured
self.crawler = crawler
self._parsers = {}
self._spider_netlocs = {}
self._useragents = {}
dispatcher.connect(self.spider_opened, signals.spider_opened)
dispatcher.connect(self.spider_closed, signals.spider_closed)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def process_request(self, request, spider):
useragent = self._useragents[spider]
rp = self.robot_parser(request, spider)
@ -42,7 +45,7 @@ class RobotsTxtMiddleware(object):
self._parsers[netloc] = None
robotsurl = "%s://%s/robots.txt" % (url.scheme, url.netloc)
robotsreq = Request(robotsurl, priority=self.DOWNLOAD_PRIORITY)
dfd = crawler.engine.download(robotsreq, spider)
dfd = self.crawler.engine.download(robotsreq, spider)
dfd.addCallback(self._parse_robots)
self._spider_netlocs[spider].add(netloc)
return self._parsers[netloc]

25
scrapy/contrib/memusage.py
View File

@ -12,31 +12,34 @@ from twisted.internet import task
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy import log
from scrapy.project import crawler
from scrapy.exceptions import NotConfigured
from scrapy.mail import MailSender
from scrapy.conf import settings
from scrapy.stats import stats
from scrapy.utils.memory import get_vmvalue_from_procfs, procfs_supported
from scrapy.utils.engine import get_engine_status
class MemoryUsage(object):
def __init__(self):
if not settings.getbool('MEMUSAGE_ENABLED'):
def __init__(self, crawler):
if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
raise NotConfigured
if not procfs_supported():
raise NotConfigured
self.crawler = crawler
self.warned = False
self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY')
self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
self.report = settings.getbool('MEMUSAGE_REPORT')
self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY')
self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
self.report = crawler.settings.getbool('MEMUSAGE_REPORT')
self.mail = MailSender()
dispatcher.connect(self.engine_started, signal=signals.engine_started)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def get_virtual_size(self):
return get_vmvalue_from_procfs('VmSize')
@ -70,10 +73,10 @@ class MemoryUsage(object):
log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR)
if self.notify_mails:
subj = "%s terminated: memory usage exceeded %dM at %s" % \
(settings['BOT_NAME'], mem, socket.gethostname())
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
self._send_report(self.notify_mails, subj)
stats.set_value('memusage/limit_notified', 1)
crawler.stop()
self.crawler.stop()
def _check_warning(self):
if self.warned: # warn only once
@ -84,7 +87,7 @@ class MemoryUsage(object):
log.msg("Memory usage reached %dM" % mem, level=log.WARNING)
if self.notify_mails:
subj = "%s warning: memory usage reached %dM at %s" % \
(settings['BOT_NAME'], mem, socket.gethostname())
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
self._send_report(self.notify_mails, subj)
stats.set_value('memusage/warning_notified', 1)
self.warned = True

10
scrapy/contrib/pipeline/media.py
View File

@ -17,12 +17,14 @@ class MediaPipeline(object):
self.downloaded = {}
self.waiting = defaultdict(list)
def __init__(self, download_func=None):
def __init__(self, download_func=None, crawler=None):
self.spiderinfo = {}
self.download_func = download_func
if not download_func:
from scrapy.project import crawler
self.crawler = crawler
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler=crawler)
def open_spider(self, spider):
self.spiderinfo[spider] = self.SpiderInfo(spider)

6
scrapy/contrib/webservice/crawler.py
View File

@ -1,10 +1,8 @@
from scrapy.webservice import JsonRpcResource
from scrapy.project import crawler
class CrawlerResource(JsonRpcResource):
ws_name = 'crawler'
def __init__(self, _crawler=crawler):
JsonRpcResource.__init__(self)
self._target = _crawler
def __init__(self, crawler):
JsonRpcResource.__init__(self, crawler, crawler)

10
scrapy/contrib/webservice/enginestatus.py
View File

@ -1,19 +1,17 @@
from scrapy.webservice import JsonResource
from scrapy.project import crawler
from scrapy.utils.engine import get_engine_status
class EngineStatusResource(JsonResource):
ws_name = 'enginestatus'
def __init__(self, spider_name=None, _crawler=crawler):
JsonResource.__init__(self)
def __init__(self, crawler, spider_name=None):
JsonResource.__init__(self, crawler)
self._spider_name = spider_name
self.isLeaf = spider_name is not None
self._crawler = _crawler
def render_GET(self, txrequest):
status = get_engine_status(self._crawler.engine)
status = get_engine_status(self.crawler.engine)
if self._spider_name is None:
return status
for sp, st in status['spiders'].items():
@ -21,4 +19,4 @@ class EngineStatusResource(JsonResource):
return st
def getChild(self, name, txrequest):
return EngineStatusResource(name, self._crawler)
return EngineStatusResource(name, self.crawler)

5
scrapy/contrib/webservice/stats.py
View File

@ -5,6 +5,5 @@ class StatsResource(JsonRpcResource):
ws_name = 'stats'
def __init__(self, _stats=stats):
JsonRpcResource.__init__(self)
self._target = _stats
def __init__(self, crawler):
JsonRpcResource.__init__(self, crawler, stats)

12
scrapy/core/downloader/__init__.py
View File

@ -62,15 +62,15 @@ def _get_concurrency_delay(concurrency, spider, settings):
class Downloader(object):
def __init__(self, settings):
self.settings = settings
def __init__(self, crawler):
self.settings = crawler.settings
self.slots = {}
self.active = set()
self.handlers = DownloadHandlers()
self.total_concurrency = settings.getint('CONCURRENT_REQUESTS')
self.domain_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self.ip_concurrency = settings.getint('CONCURRENT_REQUESTS_PER_IP')
self.middleware = DownloaderMiddlewareManager.from_settings(settings)
self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
def fetch(self, request, spider):

12
scrapy/core/engine.py
View File

@ -51,15 +51,15 @@ class Slot(object):
class ExecutionEngine(object):
def __init__(self, settings, spider_closed_callback):
self.settings = settings
def __init__(self, crawler, spider_closed_callback):
self.settings = crawler.settings
self.slots = {}
self.running = False
self.paused = False
self.scheduler_cls = load_object(settings['SCHEDULER'])
self.downloader = Downloader(self.settings)
self.scraper = Scraper(self, self.settings)
self._concurrent_spiders = settings.getint('CONCURRENT_SPIDERS')
self.scheduler_cls = load_object(self.settings['SCHEDULER'])
self.downloader = Downloader(crawler)
self.scraper = Scraper(crawler)
self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS')
self._spider_closed_callback = spider_closed_callback
@defer.inlineCallbacks

16
scrapy/core/scraper.py
View File

@ -61,13 +61,13 @@ class Slot(object):
class Scraper(object):
def __init__(self, engine, settings):
def __init__(self, crawler):
self.slots = {}
self.spidermw = SpiderMiddlewareManager.from_settings(settings)
itemproc_cls = load_object(settings['ITEM_PROCESSOR'])
self.itemproc = itemproc_cls.from_settings(settings)
self.concurrent_items = settings.getint('CONCURRENT_ITEMS')
self.engine = engine
self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
self.itemproc = itemproc_cls.from_crawler(crawler)
self.concurrent_items = crawler.settings.getint('CONCURRENT_ITEMS')
self.crawler = crawler
@defer.inlineCallbacks
def open_spider(self, spider):
@ -143,7 +143,7 @@ class Scraper(object):
def handle_spider_error(self, _failure, request, response, spider):
exc = _failure.value
if isinstance(exc, CloseSpider):
self.engine.close_spider(spider, exc.reason or 'cancelled')
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
return
log.err(_failure, "Spider error processing %s" % request, spider=spider)
send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \
@ -166,7 +166,7 @@ class Scraper(object):
if isinstance(output, Request):
send_catch_log(signal=signals.request_received, request=output, \
spider=spider)
self.engine.crawl(request=output, spider=spider)
self.crawler.engine.crawl(request=output, spider=spider)
elif isinstance(output, BaseItem):
self.slots[spider].itemproc_size += 1
dfd = self.itemproc.process_item(output, spider)

4
scrapy/crawler.py
View File

@ -31,10 +31,10 @@ class Crawler(object):
if self.configured:
return
self.configured = True
self.extensions = ExtensionManager.from_settings(self.settings)
self.extensions = ExtensionManager.from_crawler(self)
spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
self.spiders = spman_cls.from_settings(self.settings)
self.engine = ExecutionEngine(self.settings, self._spider_closed)
self.engine = ExecutionEngine(self, self._spider_closed)
def crawl(self, spider, requests=None):
spider.set_crawler(self)

10
scrapy/middleware.py
View File

@ -21,13 +21,15 @@ class MiddlewareManager(object):
raise NotImplementedError
@classmethod
def from_settings(cls, settings):
def from_settings(cls, settings, crawler=None):
mwlist = cls._get_mwlist_from_settings(settings)
middlewares = []
for clspath in mwlist:
try:
mwcls = load_object(clspath)
if hasattr(mwcls, 'from_settings'):
if crawler and hasattr(mwcls, 'from_crawler'):
mw = mwcls.from_crawler(crawler)
elif hasattr(mwcls, 'from_settings'):
mw = mwcls.from_settings(settings)
else:
mw = mwcls()
@ -41,6 +43,10 @@ class MiddlewareManager(object):
level=log.DEBUG)
return cls(*middlewares)
@classmethod
def from_crawler(cls, crawler):
return cls.from_settings(crawler.settings, crawler)
def _add_middleware(self, mw):
if hasattr(mw, 'open_spider'):
self.methods['open_spider'].append(mw.open_spider)

12
scrapy/project.py
View File

@ -1,11 +1,13 @@
"""
This module contains the Scrapy Crawler once installed by calling the crawler
``install`` method, like this::
--------- WARNING: THIS MODULE IS DEPRECATED -----------
crawler.install()
This module is deprecated. If you want to get the Scrapy crawler from your
extension, middleware or pipeline implement the `from_crawler` class method.
After that, you can import the (singleton) crawler like this::
For example:
from scrapy.project import crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
"""

27
scrapy/telnet.py
View File

@ -12,14 +12,12 @@ from twisted.internet import protocol
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import NotConfigured
from scrapy.project import crawler
from scrapy.stats import stats
from scrapy import log, signals
from scrapy.utils.signal import send_catch_log
from scrapy.utils.trackref import print_live_refs
from scrapy.utils.engine import print_engine_status
from scrapy.utils.reactor import listen_tcp
from scrapy.conf import settings
try:
import guppy
@ -34,15 +32,20 @@ update_telnet_vars = object()
class TelnetConsole(protocol.ServerFactory):
def __init__(self):
if not settings.getbool('TELNETCONSOLE_ENABLED'):
def __init__(self, crawler):
if not crawler.settings.getbool('TELNETCONSOLE_ENABLED'):
raise NotConfigured
self.crawler = crawler
self.noisy = False
self.portrange = map(int, settings.getlist('TELNETCONSOLE_PORT'))
self.host = settings['TELNETCONSOLE_HOST']
self.portrange = map(int, crawler.settings.getlist('TELNETCONSOLE_PORT'))
self.host = crawler.settings['TELNETCONSOLE_HOST']
dispatcher.connect(self.start_listening, signals.engine_started)
dispatcher.connect(self.stop_listening, signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def start_listening(self):
self.port = listen_tcp(self.portrange, self.host, self)
h = self.port.getHost()
@ -59,13 +62,13 @@ class TelnetConsole(protocol.ServerFactory):
def _get_telnet_vars(self):
# Note: if you add entries here also update topics/telnetconsole.rst
telnet_vars = {
'engine': crawler.engine,
'manager': crawler,
'extensions': crawler.extensions,
'engine': self.crawler.engine,
'manager': self.crawler,
'extensions': self.crawler.extensions,
'stats': stats,
'spiders': crawler.spiders,
'settings': settings,
'est': print_engine_status,
'spiders': self.crawler.spiders,
'settings': self.crawler.settings,
'est': lambda x: print_engine_status(self.crawler),
'p': pprint.pprint,
'prefs': print_live_refs,
'hpy': hpy,

9
scrapy/utils/engine.py
View File

@ -2,13 +2,8 @@
from time import time # used in global tests code
from scrapy.project import crawler
def get_engine_status(engine=None):
def get_engine_status(engine):
"""Return a report of the current engine status"""
if engine is None:
engine = crawler.engine
global_tests = [
"time()-engine.start_time",
"engine.has_capacity()",
@ -59,6 +54,6 @@ def format_engine_status(engine=None):
s += " %-50s : %s\n" % (test, result)
return s
def print_engine_status(engine=None):
def print_engine_status(engine):
print format_engine_status(engine)

7
scrapy/utils/serialize.py
View File

@ -19,9 +19,7 @@ class SpiderReferencer(object):
spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$')
def __init__(self, crawler=None):
if crawler is None:
from scrapy.project import crawler
def __init__(self, crawler):
self.crawler = crawler
def get_reference_from_spider(self, spider):
@ -80,7 +78,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
TIME_FORMAT = "%H:%M:%S"
def __init__(self, *a, **kw):
self.spref = kw.pop('spref', None) or SpiderReferencer()
crawler = kw.pop('crawler', None)
self.spref = kw.pop('spref', None) or SpiderReferencer(crawler)
super(ScrapyJSONEncoder, self).__init__(*a, **kw)
def encode(self, o):

37
scrapy/webservice.py
View File

@ -15,19 +15,21 @@ from scrapy.utils.misc import load_object
from scrapy.utils.txweb import JsonResource as JsonResource_
from scrapy.utils.reactor import listen_tcp
from scrapy.utils.conf import build_component_list
from scrapy.conf import settings
class JsonResource(JsonResource_):
json_encoder = ScrapyJSONEncoder()
def __init__(self, crawler, target=None):
JsonResource_.__init__(self)
self.crawler = crawler
self.json_encoder = ScrapyJSONEncoder(crawler=crawler)
class JsonRpcResource(JsonResource):
json_decoder = ScrapyJSONDecoder()
def __init__(self, target=None):
JsonResource.__init__(self)
def __init__(self, crawler, target=None):
JsonResource.__init__(self, crawler, target)
self.json_decoder = ScrapyJSONDecoder(crawler=crawler)
self.crawler = crawler
self._target = target
def render_GET(self, txrequest):
@ -63,23 +65,28 @@ class RootResource(JsonResource):
class WebService(server.Site):
def __init__(self):
if not settings.getbool('WEBSERVICE_ENABLED'):
def __init__(self, crawler):
if not crawler.settings.getbool('WEBSERVICE_ENABLED'):
raise NotConfigured
logfile = settings['WEBSERVICE_LOGFILE']
self.portrange = map(int, settings.getlist('WEBSERVICE_PORT'))
self.host = settings['WEBSERVICE_HOST']
root = RootResource()
reslist = build_component_list(settings['WEBSERVICE_RESOURCES_BASE'], \
settings['WEBSERVICE_RESOURCES'])
self.crawler = crawler
logfile = crawler.settings['WEBSERVICE_LOGFILE']
self.portrange = map(int, crawler.settings.getlist('WEBSERVICE_PORT'))
self.host = crawler.settings['WEBSERVICE_HOST']
root = RootResource(crawler)
reslist = build_component_list(crawler.settings['WEBSERVICE_RESOURCES_BASE'], \
crawler.settings['WEBSERVICE_RESOURCES'])
for res_cls in map(load_object, reslist):
res = res_cls()
res = res_cls(crawler)
root.putChild(res.ws_name, res)
server.Site.__init__(self, root, logPath=logfile)
self.noisy = False
dispatcher.connect(self.start_listening, signals.engine_started)
dispatcher.connect(self.stop_listening, signals.engine_stopped)
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def start_listening(self):
self.port = listen_tcp(self.portrange, self.host, self)
h = self.port.getHost()