diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index a2b0edc6b..c7e859515 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -7,6 +7,7 @@ import cProfile import scrapy from scrapy import log +from scrapy.crawler import CrawlerProcess from scrapy.xlib import lsprofcalltree from scrapy.conf import settings from scrapy.command import ScrapyCommand @@ -79,6 +80,8 @@ def check_deprecated_scrapy_ctl(argv): def execute(argv=None): if argv is None: argv = sys.argv + crawler = CrawlerProcess(settings) + crawler.install() check_deprecated_scrapy_ctl(argv) # TODO: remove for Scrapy 0.11 cmds = _get_commands_dict() cmdname = _get_command_name(argv) @@ -113,9 +116,8 @@ def execute(argv=None): settings.defaults.update(cmd.default_settings) del args[0] # remove command name from args - from scrapy.project import crawler log.start() - crawler.configure() + cmd.set_crawler(crawler) ret = _run_command(cmd, args, opts) if ret is False: parser.print_help() diff --git a/scrapy/command.py b/scrapy/command.py index ab357c08f..fecbeab01 100644 --- a/scrapy/command.py +++ b/scrapy/command.py @@ -18,6 +18,14 @@ class ScrapyCommand(object): # default settings to be used for this command instead of global defaults default_settings = {} + def set_crawler(self, crawler): + self._crawler = crawler + + @property + def crawler(self): + self._crawler.configure() + return self._crawler + def syntax(self): """ Command syntax (preferably one-line). Do not include command name. diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py index 7a7d76edc..db6009206 100644 --- a/scrapy/commands/crawl.py +++ b/scrapy/commands/crawl.py @@ -1,10 +1,7 @@ from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.core.queue import ExecutionQueue -from scrapy.project import crawler from scrapy.conf import settings from scrapy.http import Request -from scrapy.project import crawler from scrapy.utils.url import is_url from collections import defaultdict @@ -32,31 +29,29 @@ class Command(ScrapyCommand): settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False def run(self, args, opts): - q = ExecutionQueue() + q = self.crawler.queue urls, names = self._split_urls_and_names(args) for name in names: q.append_spider_name(name) if opts.spider: try: - spider = crawler.spiders.create(opts.spider) + spider = self.crawler.spiders.create(opts.spider) for url in urls: q.append_url(url, spider) except KeyError: log.msg('Unable to find spider: %s' % opts.spider, log.ERROR) else: for name, urls in self._group_urls_by_spider(urls): - spider = crawler.spiders.create(name) + spider = self.crawler.spiders.create(name) for url in urls: q.append_url(url, spider) - - crawler.queue = q - crawler.start() + self.crawler.start() def _group_urls_by_spider(self, urls): spider_urls = defaultdict(list) for url in urls: - spider_names = crawler.spiders.find_by_request(Request(url)) + spider_names = self.crawler.spiders.find_by_request(Request(url)) if not spider_names: log.msg('Could not find spider that handles url: %s' % url, log.ERROR) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index eefbd71eb..980348b12 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -2,10 +2,8 @@ import pprint from scrapy import log from scrapy.command import ScrapyCommand -from scrapy.project import crawler from scrapy.http import Request from scrapy.spider import BaseSpider -from scrapy.project import crawler from scrapy.utils.url import is_url class Command(ScrapyCommand): @@ -44,11 +42,11 @@ class Command(ScrapyCommand): spider = None if opts.spider: try: - spider = crawler.spiders.create(opts.spider) + spider = self.crawler.spiders.create(opts.spider) except KeyError: log.msg("Could not find spider: %s" % opts.spider, log.ERROR) - crawler.queue.append_request(request, spider, \ + self.crawler.queue.append_request(request, spider, \ default_spider=BaseSpider('default')) - crawler.start() + self.crawler.start() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index f451afbc4..35bb0f23e 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -4,7 +4,6 @@ from os import listdir from os.path import join, dirname, abspath, exists, splitext import scrapy -from scrapy.project import crawler from scrapy.command import ScrapyCommand from scrapy.conf import settings from scrapy.utils.template import render_templatefile, string_camelcase @@ -60,7 +59,7 @@ class Command(ScrapyCommand): name, domain = args[0:2] module = sanitize_module_name(name) try: - spider = crawler.spiders.create(name) + spider = self.crawler.spiders.create(name) except KeyError: pass else: diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index a3fe8a3fa..29b57edfd 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,5 +1,4 @@ from scrapy.command import ScrapyCommand -from scrapy.project import crawler class Command(ScrapyCommand): @@ -10,4 +9,4 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - print "\n".join(crawler.spiders.list()) + print "\n".join(self.crawler.spiders.list()) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index a0fd193dd..839b226ab 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,5 +1,4 @@ from scrapy.command import ScrapyCommand -from scrapy.project import crawler from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils import display @@ -69,11 +68,11 @@ class Command(ScrapyCommand): def get_spider(self, request, opts): if opts.spider: try: - return crawler.spiders.create(opts.spider) + return self.crawler.spiders.create(opts.spider) except KeyError: log.msg('Unable to find spider: %s' % opts.spider, log.ERROR) else: - spider = crawler.spiders.create_for_request(request) + spider = self.crawler.spiders.create_for_request(request) if spider: return spider log.msg('Unable to find spider for: %s' % request, log.ERROR) @@ -84,8 +83,8 @@ class Command(ScrapyCommand): spider = self.get_spider(request, opts) if not spider: return None, None - crawler.queue.append_request(request, spider) - crawler.start() + self.crawler.queue.append_request(request, spider) + self.crawler.start() if not responses: log.msg('No response downloaded for: %s' % request, log.ERROR, \ spider=spider) diff --git a/scrapy/commands/queue.py b/scrapy/commands/queue.py index f1d43b0a5..03379d8f6 100644 --- a/scrapy/commands/queue.py +++ b/scrapy/commands/queue.py @@ -1,8 +1,9 @@ from scrapy.command import ScrapyCommand +from scrapy.commands import runserver from scrapy.utils.misc import load_object from scrapy.conf import settings -class Command(ScrapyCommand): +class Command(runserver.Command): requires_project = True default_settings = {'LOG_ENABLED': False} @@ -26,7 +27,7 @@ class Command(ScrapyCommand): cmd = args[0] botname = settings['BOT_NAME'] - queue = load_object(settings['SERVICE_QUEUE'])().queue + queue = self.crawler.queue.queue if cmd == 'add': if len(args) < 2: diff --git a/scrapy/commands/runserver.py b/scrapy/commands/runserver.py index 9cd6778b1..21a36fe59 100644 --- a/scrapy/commands/runserver.py +++ b/scrapy/commands/runserver.py @@ -1,6 +1,4 @@ from scrapy.command import ScrapyCommand -from scrapy.project import crawler -from scrapy.utils.misc import load_object from scrapy.conf import settings class Command(ScrapyCommand): @@ -10,7 +8,9 @@ class Command(ScrapyCommand): def short_desc(self): return "Start Scrapy in server mode" + def process_options(self, args, opts): + super(Command, self).process_options(args, opts) + settings.overrides['QUEUE_CLASS'] = settings['SERVER_QUEUE_CLASS'] + def run(self, args, opts): - queue_class = load_object(settings['SERVICE_QUEUE']) - crawler.queue = queue_class() - crawler.start() + self.crawler.start() diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index 98d80ddbb..84253c66d 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -4,7 +4,6 @@ import os from scrapy import log from scrapy.utils.spider import iter_spider_classes from scrapy.command import ScrapyCommand -from scrapy.project import crawler def _import_file(filepath): abspath = os.path.abspath(filepath) @@ -52,5 +51,5 @@ class Command(ScrapyCommand): return spider = spclasses.pop()() # schedule spider and start engine - crawler.queue.append_spider(spider) - crawler.start() + self.crawler.queue.append_spider(spider) + self.crawler.start() diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 22406227f..dab5d2709 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,5 +1,4 @@ from scrapy.command import ScrapyCommand -from scrapy.conf import settings as settings_ class Command(ScrapyCommand): @@ -24,26 +23,16 @@ class Command(ScrapyCommand): help="print setting value, intepreted as an float") parser.add_option("--getlist", dest="getlist", metavar="SETTING", \ help="print setting value, intepreted as an float") - parser.add_option("--init", dest="init", action="store_true", \ - help="print initial setting value (before loading extensions and spiders)") - - def process_options(self, args, opts): - super(Command, self).process_options(args, opts) - if opts.init: - self._print_setting(opts) def run(self, args, opts): - if not opts.init: - self._print_setting(opts) - - def _print_setting(self, opts): + settings = self.crawler.settings if opts.get: - print settings_.get(opts.get) + print settings.get(opts.get) elif opts.getbool: - print settings_.getbool(opts.getbool) + print settings.getbool(opts.getbool) elif opts.getint: - print settings_.getint(opts.getint) + print settings.getint(opts.getint) elif opts.getfloat: - print settings_.getfloat(opts.getfloat) + print settings.getfloat(opts.getfloat) elif opts.getlist: - print settings_.getlist(opts.getlist) + print settings.getlist(opts.getlist) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 2d0e04d45..87119718c 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,14 +4,13 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -from scrapy.project import crawler -from scrapy.core.queue import KeepAliveExecutionQueue from scrapy.command import ScrapyCommand from scrapy.shell import Shell class Command(ScrapyCommand): requires_project = False + default_settings = {'QUEUE_CLASS': 'scrapy.core.queue.KeepAliveExecutionQueue'} def syntax(self): return "[url|file]" @@ -30,7 +29,6 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None - shell = Shell(crawler, update_vars=self.update_vars, inthread=True) - shell.start(url=url).addBoth(lambda _: crawler.stop()) - crawler.queue = KeepAliveExecutionQueue() - crawler.start() + shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True) + shell.start(url=url).addBoth(lambda _: self.crawler.stop()) + self.crawler.start() diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 70222f268..6b03ca43f 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -175,6 +175,8 @@ MEMUSAGE_WARNING_MB = 0 NEWSPIDER_MODULE = '' +QUEUE_CLASS = 'scrapy.core.queue.ExecutionQueue' + RANDOMIZE_DOWNLOAD_DELAY = True REDIRECT_MAX_METAREFRESH_DELAY = 100 @@ -207,8 +209,7 @@ SCHEDULER_MIDDLEWARES_BASE = { SCHEDULER_ORDER = 'DFO' -SERVICE_QUEUE = 'scrapy.contrib.queue.SqliteExecutionQueue' -SERVICE_QUEUE_FILE = 'scrapy.db' +SERVER_QUEUE_CLASS = 'scrapy.contrib.queue.SqliteExecutionQueue' SPIDER_MANAGER_CLASS = 'scrapy.spidermanager.SpiderManager' @@ -227,6 +228,8 @@ SPIDER_MIDDLEWARES_BASE = { SPIDER_MODULES = [] +SQLITE_DB = 'scrapy.db' + SQS_QUEUE = 'scrapy' SQS_VISIBILITY_TIMEOUT = 7200 SQS_POLLING_DELAY = 30 diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 752a3c144..0c229376b 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -2,7 +2,6 @@ from twisted.internet.defer import Deferred, DeferredList from scrapy.utils.defer import mustbe_deferred, defer_result from scrapy import log -from scrapy.project import crawler from scrapy.utils.request import request_fingerprint from scrapy.utils.misc import arg_to_iter @@ -21,6 +20,8 @@ class MediaPipeline(object): def __init__(self): self.spiderinfo = {} + from scrapy.project import crawler + self.crawler = crawler def open_spider(self, spider): self.spiderinfo[spider] = self.SpiderInfo(spider) diff --git a/scrapy/contrib/queue/__init__.py b/scrapy/contrib/queue/__init__.py index fe39f07ce..dc3d46db8 100644 --- a/scrapy/contrib/queue/__init__.py +++ b/scrapy/contrib/queue/__init__.py @@ -8,7 +8,7 @@ class SqliteExecutionQueue(ExecutionQueue): def __init__(self, *a, **kw): super(SqliteExecutionQueue, self).__init__(*a, **kw) - self.queue = JsonSqlitePriorityQueue(settings['SERVICE_QUEUE_FILE']) + self.queue = JsonSqlitePriorityQueue(settings['SQLITE_DB']) def _append_next(self): msg = self.queue.pop() diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index f7fe32254..6516fed89 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -40,7 +40,7 @@ class ExecutionEngine(object): """ self.scheduler = load_object(settings['SCHEDULER'])() self.downloader = Downloader() - self.scraper = Scraper(self) + self.scraper = Scraper(self, self.crawler.settings) self.configured = True self._spider_closed_callback = spider_closed_callback diff --git a/scrapy/core/queue.py b/scrapy/core/queue.py index 428e7a5ab..17ba27272 100644 --- a/scrapy/core/queue.py +++ b/scrapy/core/queue.py @@ -9,10 +9,7 @@ class ExecutionQueue(object): polling_delay = 5 - def __init__(self, _spiders=None): - if _spiders is None: - from scrapy.project import crawler - _spiders = crawler.spiders + def __init__(self, _spiders): self.spider_requests = [] self._spiders = _spiders diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1fd5fc05a..50be2ed5c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -15,7 +15,6 @@ from scrapy.item import BaseItem from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy import log from scrapy.stats import stats -from scrapy.conf import settings class SpiderInfo(object): @@ -60,7 +59,7 @@ class SpiderInfo(object): class Scraper(object): - def __init__(self, engine): + def __init__(self, engine, settings): self.sites = {} self.spidermw = SpiderMiddlewareManager() itemproc_cls = load_object(settings['ITEM_PROCESSOR']) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5e715b37f..e6793d369 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,6 @@ from twisted.internet import reactor, defer from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import ExecutionEngine -from scrapy.core.queue import ExecutionQueue from scrapy.extension import ExtensionManager from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object @@ -16,17 +15,28 @@ class Crawler(object): def __init__(self, settings): self.configured = False self.settings = settings - self.spiders = load_object(settings['SPIDER_MANAGER_CLASS'])() - self.engine = ExecutionEngine(self) - def configure(self, queue=None): - self.extensions = ExtensionManager.from_settings(self.settings) - if not self.spiders.loaded: - self.spiders.load() + def install(self): + import scrapy.project + assert not hasattr(scrapy.project, 'crawler'), "crawler already installed" + scrapy.project.crawler = self - self.queue = queue or ExecutionQueue() - self.engine.configure(self._spider_closed) + def uninstall(self): + import scrapy.project + assert hasattr(scrapy.project, 'crawler'), "crawler not installed" + del scrapy.project.crawler + + def configure(self): + if self.configured: + return self.configured = True + self.engine = ExecutionEngine(self) + self.extensions = ExtensionManager.from_settings(self.settings) + spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS']) + self.spiders = spman_cls.from_settings(self.settings) + queue_cls = load_object(self.settings['QUEUE_CLASS']) + self.queue = queue_cls(self.spiders) + self.engine.configure(self._spider_closed) @defer.inlineCallbacks def _start_next_spider(self): @@ -56,6 +66,7 @@ class Crawler(object): @defer.inlineCallbacks def start(self): + yield defer.maybeDeferred(self.configure) yield defer.maybeDeferred(self.engine.start) self._nextcall = reactor.callLater(0, self._start_next_spider) diff --git a/scrapy/project.py b/scrapy/project.py index 5cb52b9b2..bbcb0fd27 100644 --- a/scrapy/project.py +++ b/scrapy/project.py @@ -1,4 +1,11 @@ -from scrapy.crawler import CrawlerProcess -from scrapy.conf import settings +""" +This module contains the Scrapy Crawler once installed by calling the crawler +``install`` method, like this:: -crawler = CrawlerProcess(settings) + crawler.install() + +After that, you can import the (singleton) crawler like this:: + + from scrapy.project import crawler + +""" diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py index edc17e40e..0b0a3c673 100644 --- a/scrapy/spidermanager.py +++ b/scrapy/spidermanager.py @@ -4,16 +4,26 @@ spiders """ from scrapy import log -from scrapy.conf import settings from scrapy.utils.misc import walk_modules from scrapy.utils.spider import iter_spider_classes class SpiderManager(object): - def __init__(self): - self.loaded = False + def __init__(self, spider_modules): + self.spider_modules = spider_modules self._spiders = {} + for name in self.spider_modules: + for module in walk_modules(name): + self._load_spiders(module) + + def _load_spiders(self, module): + for spcls in iter_spider_classes(module): + self._spiders[spcls.name] = spcls + + @classmethod + def from_settings(cls, settings): + return cls(settings.getlist('SPIDER_MODULES')) def create(self, spider_name, **spider_kwargs): """Returns a Spider instance for the given spider name, using the given @@ -53,21 +63,5 @@ class SpiderManager(object): """Returns list of spiders available.""" return self._spiders.keys() - def load(self, spider_modules=None): - """Load spiders from spider_modules or SPIDER_MODULES setting.""" - if spider_modules is None: - spider_modules = settings.getlist('SPIDER_MODULES') - self.spider_modules = spider_modules - - self._spiders = {} - for name in self.spider_modules: - for module in walk_modules(name): - self._load_spiders(module) - self.loaded = True - - def _load_spiders(self, module): - for spcls in iter_spider_classes(module): - self._spiders[spcls.name] = spcls - def close_spider(self, spider): pass diff --git a/scrapy/telnet.py b/scrapy/telnet.py index f4ae9673d..41b17ac74 100644 --- a/scrapy/telnet.py +++ b/scrapy/telnet.py @@ -10,9 +10,11 @@ from twisted.conch import manhole, telnet from twisted.conch.insults import insults from twisted.internet import reactor, protocol +from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured from scrapy.project import crawler from scrapy.stats import stats +from scrapy import signals from scrapy.utils.signal import send_catch_log from scrapy.utils.trackref import print_live_refs from scrapy.utils.engine import print_engine_status @@ -35,8 +37,15 @@ class TelnetConsole(protocol.ServerFactory): if not settings.getbool('TELNETCONSOLE_ENABLED'): raise NotConfigured self.noisy = False - port = settings.getint('TELNETCONSOLE_PORT') - reactor.callWhenRunning(reactor.listenTCP, port, self) + self.portnum = settings.getint('TELNETCONSOLE_PORT') + dispatcher.connect(self.start_listening, signals.engine_started) + dispatcher.connect(self.stop_listening, signals.engine_stopped) + + def start_listening(self): + self.port = reactor.listenTCP(self.portnum, self) + + def stop_listening(self): + self.port.stopListening() def protocol(self): telnet_vars = self._get_telnet_vars() diff --git a/scrapy/tests/test_cmdline/__init__.py b/scrapy/tests/test_cmdline/__init__.py index 7c88870c3..697ebb512 100644 --- a/scrapy/tests/test_cmdline/__init__.py +++ b/scrapy/tests/test_cmdline/__init__.py @@ -20,29 +20,20 @@ class CmdlineTest(unittest.TestCase): return comm[0].strip() def test_default_settings(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init'), \ - 'default') self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ 'default + loaded + started') def test_override_settings_using_settings_arg(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init', \ - '--settings', 'scrapy.tests.test_cmdline.settings2'), \ - 'override') self.assertEqual(self._execute('settings', '--get', 'TEST1', \ '--settings', 'scrapy.tests.test_cmdline.settings2'), \ 'override + loaded + started') def test_override_settings_using_set_arg(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init', '--set', 'TEST1=override'), \ - 'override') self.assertEqual(self._execute('settings', '--get', 'TEST1', '--set', 'TEST1=override'), \ 'override + loaded + started') def test_override_settings_using_envvar(self): self.env['SCRAPY_TEST1'] = 'override' - self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init'), \ - 'override') self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ 'override + loaded + started') diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 79de27fe3..160193d3f 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -1,20 +1,19 @@ -""" -Scrapy engine tests -""" +import sys, os, re, urlparse -import sys, os, re, urlparse, unittest - -from twisted.internet import reactor -from twisted.web import server, resource, static, util +from twisted.internet import reactor, defer +from twisted.web import server, static, util +from twisted.trial import unittest from scrapy import signals -from scrapy.project import crawler +from scrapy.conf import Settings +from scrapy.crawler import Crawler from scrapy.xlib.pydispatch import dispatcher from scrapy.tests import tests_datadir from scrapy.spider import BaseSpider from scrapy.item import Item, Field from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy.http import Request +from scrapy.utils.signal import disconnect_all class TestItem(Item): name = Field() @@ -24,7 +23,6 @@ class TestItem(Item): class TestSpider(BaseSpider): name = "scrapytest.org" allowed_domains = ["scrapytest.org", "localhost"] - start_urls = ['http://localhost'] itemurl_re = re.compile("item\d+.html") name_re = re.compile("

(.*?)

", re.M) @@ -48,66 +46,61 @@ class TestSpider(BaseSpider): item['price'] = m.group(1) return item -#class TestResource(resource.Resource): -# isLeaf = True -# -# def render_GET(self, request): -# return "hello world!" - -def start_test_site(): +def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) -# r.putChild("test", TestResource()) r.putChild("redirect", util.Redirect("/redirected")) r.putChild("redirected", static.Data("Redirected here", "text/plain")) port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") + if debug: + print "Test server running at http://localhost:%d/ - hit Ctrl-C to finish." \ + % port.getHost().port return port -class CrawlingSession(object): +class CrawlerRun(object): + """A class to run the crawler and keep track of events occurred""" def __init__(self): - self.name = 'scrapytest.org' self.spider = None self.respplug = [] self.reqplug = [] self.itemresp = [] self.signals_catched = {} - self.wasrun = False def run(self): self.port = start_test_site() self.portno = self.port.getHost().port - self.spider = TestSpider() - if self.spider: - self.spider.start_urls = [ - self.geturl("/"), - self.geturl("/redirect"), - ] + start_urls = [self.geturl("/"), self.geturl("/redirect")] + self.spider = TestSpider(start_urls=start_urls) - dispatcher.connect(self.record_signal, signals.engine_started) - dispatcher.connect(self.record_signal, signals.engine_stopped) - dispatcher.connect(self.record_signal, signals.spider_opened) - dispatcher.connect(self.record_signal, signals.spider_idle) - dispatcher.connect(self.record_signal, signals.spider_closed) - dispatcher.connect(self.item_scraped, signals.item_scraped) - dispatcher.connect(self.request_received, signals.request_received) - dispatcher.connect(self.response_downloaded, signals.response_downloaded) + for name, signal in vars(signals).items(): + if not name.startswith('_'): + dispatcher.connect(self.record_signal, signal) + dispatcher.connect(self.item_scraped, signals.item_scraped) + dispatcher.connect(self.request_received, signals.request_received) + dispatcher.connect(self.response_downloaded, signals.response_downloaded) - crawler.configure() - crawler.queue.append_spider(self.spider) - crawler.start() - self.port.stopListening() - self.wasrun = True - # FIXME: extremly ugly hack to avoid propagating errors to other - # stats because of living signals. This whole test_engine.py should - # be rewritten from scratch actually. - from scrapy.utils.signal import disconnect_all - disconnect_all(signals.stats_spider_opened) - disconnect_all(signals.stats_spider_closing) - disconnect_all(signals.stats_spider_closed) + settings = Settings() + self.crawler = Crawler(settings) + self.crawler.install() + self.crawler.configure() + self.crawler.queue.append_spider(self.spider) + self.crawler.start() + + self.deferred = defer.Deferred() + dispatcher.connect(self.stop, signals.engine_stopped) + return self.deferred + + def stop(self): + self.port.stopListening() + for name, signal in vars(signals).items(): + if not name.startswith('_'): + disconnect_all(signal) + self.crawler.uninstall() + self.deferred.callback(None) def geturl(self, path): return "http://localhost:%s%s" % (self.portno, path) @@ -132,67 +125,49 @@ class CrawlingSession(object): signalargs.pop('sender', None) self.signals_catched[sig] = signalargs -session = CrawlingSession() - class EngineTest(unittest.TestCase): - def setUp(self): - if not session.wasrun: - session.run() + @defer.inlineCallbacks + def test_crawler(self): + self.run = CrawlerRun() + yield self.run.run() + self._assert_visited_urls() + self._assert_received_requests() + self._assert_downloaded_responses() + self._assert_scraped_items() + self._assert_signals_catched() - def test_spider_locator(self): - """ - Check the spider is loaded and located properly via the SpiderLocator - """ - assert session.spider is not None - self.assertEqual(session.spider.name, session.name) - - def test_visited_urls(self): - """ - Make sure certain URls were actually visited - """ - # expected urls that should be visited + def _assert_visited_urls(self): must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] - - urls_visited = set([rp[0].url for rp in session.respplug]) - urls_expected = set([session.geturl(p) for p in must_be_visited]) + urls_visited = set([rp[0].url for rp in self.run.respplug]) + urls_expected = set([self.run.geturl(p) for p in must_be_visited]) assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) - def test_requests_received(self): - """ - Check requests received - """ + def _assert_received_requests(self): # 3 requests should be received from the spider. start_urls and redirects don't count - self.assertEqual(3, len(session.reqplug)) + self.assertEqual(3, len(self.run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html'] - urls_requested = set([rq[0].url for rq in session.reqplug]) - urls_expected = set([session.geturl(p) for p in paths_expected]) + urls_requested = set([rq[0].url for rq in self.run.reqplug]) + urls_expected = set([self.run.geturl(p) for p in paths_expected]) assert urls_expected <= urls_requested - def test_responses_downloaded(self): - """ - Check responses downloaded - """ + def _assert_downloaded_responses(self): # response tests - self.assertEqual(6, len(session.respplug)) + self.assertEqual(6, len(self.run.respplug)) - for response, spider in session.respplug: - if session.getpath(response.url) == '/item999.html': + for response, _ in self.run.respplug: + if self.run.getpath(response.url) == '/item999.html': self.assertEqual(404, response.status) - if session.getpath(response.url) == '/redirect': + if self.run.getpath(response.url) == '/redirect': self.assertEqual(302, response.status) - def test_item_data(self): - """ - Check item data - """ - # item tests - self.assertEqual(2, len(session.itemresp)) - for item, response in session.itemresp: + def _assert_scraped_items(self): + self.assertEqual(2, len(self.run.itemresp)) + for item, response in self.run.itemresp: self.assertEqual(item['url'], response.url) if 'item1.html' in item['url']: self.assertEqual('Item 1 name', item['name']) @@ -201,29 +176,22 @@ class EngineTest(unittest.TestCase): self.assertEqual('Item 2 name', item['name']) self.assertEqual('200', item['price']) - def test_signals(self): - """ - Check signals were sent properly - """ - from scrapy import signals + def _assert_signals_catched(self): + assert signals.engine_started in self.run.signals_catched + assert signals.engine_stopped in self.run.signals_catched + assert signals.spider_opened in self.run.signals_catched + assert signals.spider_idle in self.run.signals_catched + assert signals.spider_closed in self.run.signals_catched - assert signals.engine_started in session.signals_catched - assert signals.engine_stopped in session.signals_catched - assert signals.spider_opened in session.signals_catched - assert signals.spider_idle in session.signals_catched - assert signals.spider_closed in session.signals_catched + self.assertEqual({'spider': self.run.spider}, + self.run.signals_catched[signals.spider_opened]) + self.assertEqual({'spider': self.run.spider}, + self.run.signals_catched[signals.spider_idle]) + self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, + self.run.signals_catched[signals.spider_closed]) - self.assertEqual({'spider': session.spider}, - session.signals_catched[signals.spider_opened]) - self.assertEqual({'spider': session.spider}, - session.signals_catched[signals.spider_idle]) - self.assertEqual({'spider': session.spider, 'reason': 'finished'}, - session.signals_catched[signals.spider_closed]) if __name__ == "__main__": if len(sys.argv) > 1 and sys.argv[1] == 'runserver': - port = start_test_site() - print "Test server running at http://localhost:%d/ - hit Ctrl-C to finish." % port.getHost().port + start_test_site(debug=True) reactor.run() - else: - unittest.main() diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index 49d44bba4..5d58abd2c 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -1,5 +1,6 @@ import os from twisted.trial import unittest +from scrapy.crawler import Crawler from scrapy.conf import settings from tempfile import mkdtemp from shutil import rmtree @@ -7,6 +8,8 @@ from shutil import rmtree class ImagesPipelineTestCase(unittest.TestCase): def setUp(self): + self.crawler = Crawler(settings) + self.crawler.install() try: import Image except ImportError, e: @@ -23,6 +26,7 @@ class ImagesPipelineTestCase(unittest.TestCase): del self.pipeline rmtree(self.tempdir) settings.disabled = self.settings_disabled_before + self.crawler.uninstall() def test_image_path(self): image_path = self.pipeline.image_key diff --git a/scrapy/tests/test_pipeline_media.py b/scrapy/tests/test_pipeline_media.py index d1bd62aac..6877fcfad 100644 --- a/scrapy/tests/test_pipeline_media.py +++ b/scrapy/tests/test_pipeline_media.py @@ -2,6 +2,8 @@ from twisted.trial import unittest from twisted.python import failure from twisted.internet import defer, reactor +from scrapy.conf import Settings +from scrapy.crawler import Crawler from scrapy.http import Request, Response from scrapy.spider import BaseSpider from scrapy.utils.request import request_fingerprint @@ -29,12 +31,15 @@ class MediaPipelineTestCase(unittest.TestCase): pipeline_class = _MockedMediaPipeline def setUp(self): + self.crawler = Crawler(Settings()) + self.crawler.install() self.spider = BaseSpider('media.com') self.pipe = self.pipeline_class() self.pipe.open_spider(self.spider) def tearDown(self): self.pipe.close_spider(self.spider) + self.crawler.uninstall() @defer.inlineCallbacks def test_return_item_by_default(self): diff --git a/scrapy/tests/test_spidermanager/__init__.py b/scrapy/tests/test_spidermanager/__init__.py index 4b555eb2a..8bee3c4e1 100644 --- a/scrapy/tests/test_spidermanager/__init__.py +++ b/scrapy/tests/test_spidermanager/__init__.py @@ -22,10 +22,7 @@ class SpiderManagerTest(unittest.TestCase): self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx') shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) - self.spiderman = SpiderManager() - assert not self.spiderman.loaded - self.spiderman.load(['test_spiders_xxx']) - assert self.spiderman.loaded + self.spiderman = SpiderManager(['test_spiders_xxx']) def tearDown(self): del self.spiderman @@ -57,9 +54,9 @@ class SpiderManagerTest(unittest.TestCase): ['spider3']) def test_load_spider_module(self): - self.spiderman.load(['scrapy.tests.test_spidermanager.test_spiders.spider1']) + self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider1']) assert len(self.spiderman._spiders) == 1 def test_load_base_spider(self): - self.spiderman.load(['scrapy.tests.test_spidermanager.test_spiders.spider0']) + self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0']) assert len(self.spiderman._spiders) == 0 diff --git a/scrapy/tests/test_utils_jsonrpc.py b/scrapy/tests/test_utils_jsonrpc.py index 5e2e01344..6bb9a599f 100644 --- a/scrapy/tests/test_utils_jsonrpc.py +++ b/scrapy/tests/test_utils_jsonrpc.py @@ -3,6 +3,8 @@ from cStringIO import StringIO from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \ JsonRpcError, jsonrpc_errors +from scrapy.utils.serialize import ScrapyJSONDecoder +from scrapy.tests.test_utils_serialize import ExecutionMangerStub from scrapy.utils.py26 import json class urllib_stub(object): @@ -30,6 +32,10 @@ class TestTarget(object): class JsonRpcUtilsTestCase(unittest.TestCase): + def setUp(self): + crawler = ExecutionMangerStub([]) + self.json_decoder = ScrapyJSONDecoder(crawler=crawler) + def test_jsonrpc_client_call_request(self): ul = urllib_stub(1) jsonrpc_client_call('url', 'test', 'one', 2, _urllib=ul) @@ -62,26 +68,26 @@ class JsonRpcUtilsTestCase(unittest.TestCase): def test_jsonrpc_server_call(self): t = TestTarget() - r = jsonrpc_server_call(t, 'invalid json data') + r = jsonrpc_server_call(t, 'invalid json data', self.json_decoder) assert 'error' in r assert r['jsonrpc'] == '2.0' assert r['id'] is None self.assertEqual(r['error']['code'], jsonrpc_errors.PARSE_ERROR) assert 'Traceback' in r['error']['data'] - r = jsonrpc_server_call(t, '{"test": "test"}') + r = jsonrpc_server_call(t, '{"test": "test"}', self.json_decoder) assert 'error' in r assert r['jsonrpc'] == '2.0' assert r['id'] is None self.assertEqual(r['error']['code'], jsonrpc_errors.INVALID_REQUEST) - r = jsonrpc_server_call(t, '{"method": "notfound", "id": 1}') + r = jsonrpc_server_call(t, '{"method": "notfound", "id": 1}', self.json_decoder) assert 'error' in r assert r['jsonrpc'] == '2.0' assert r['id'] == 1 self.assertEqual(r['error']['code'], jsonrpc_errors.METHOD_NOT_FOUND) - r = jsonrpc_server_call(t, '{"method": "exception", "id": 1}') + r = jsonrpc_server_call(t, '{"method": "exception", "id": 1}', self.json_decoder) assert 'error' in r assert r['jsonrpc'] == '2.0' assert r['id'] == 1 @@ -89,19 +95,21 @@ class JsonRpcUtilsTestCase(unittest.TestCase): assert 'testing-errors' in r['error']['message'] assert 'Traceback' in r['error']['data'] - r = jsonrpc_server_call(t, '{"method": "call", "id": 2}') + r = jsonrpc_server_call(t, '{"method": "call", "id": 2}', self.json_decoder) assert 'result' in r assert r['jsonrpc'] == '2.0' assert r['id'] == 2 self.assertEqual(r['result'], ([], {})) - r = jsonrpc_server_call(t, '{"method": "call", "params": [456, 123], "id": 3}') + r = jsonrpc_server_call(t, '{"method": "call", "params": [456, 123], "id": 3}', \ + self.json_decoder) assert 'result' in r assert r['jsonrpc'] == '2.0' assert r['id'] == 3 self.assertEqual(r['result'], ([456, 123], {})) - r = jsonrpc_server_call(t, '{"method": "call", "params": {"data": 789}, "id": 3}') + r = jsonrpc_server_call(t, '{"method": "call", "params": {"data": 789}, "id": 3}', \ + self.json_decoder) assert 'result' in r assert r['jsonrpc'] == '2.0' assert r['id'] == 3 diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index b38963265..73f92503e 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -4,7 +4,6 @@ import decimal from twisted.internet import defer -from scrapy.project import crawler from scrapy.spider import BaseSpider from scrapy.http import Request, Response from scrapy.utils.py26 import json @@ -20,8 +19,10 @@ class SpiderReferencer(object): spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$') - def __init__(self, manager=None): - self.manager = manager or crawler + def __init__(self, crawler=None): + if crawler is None: + from scrapy.project import crawler + self.manager = crawler def get_reference_from_spider(self, spider): return 'spider:%x:%s' % (id(spider), spider.name) @@ -109,7 +110,8 @@ class ScrapyJSONEncoder(json.JSONEncoder): class ScrapyJSONDecoder(json.JSONDecoder): def __init__(self, *a, **kw): - self.spref = kw.pop('spref', None) or SpiderReferencer() + crawler = kw.pop('crawler', None) + self.spref = kw.pop('spref', None) or SpiderReferencer(crawler) super(ScrapyJSONDecoder, self).__init__(*a, **kw) def decode(self, s): diff --git a/scrapy/webservice.py b/scrapy/webservice.py index 44f804fea..6547747fc 100644 --- a/scrapy/webservice.py +++ b/scrapy/webservice.py @@ -7,7 +7,9 @@ See docs/topics/ws.rst from twisted.internet import reactor from twisted.web import server, resource, error +from scrapy.xlib.pydispatch import dispatcher from scrapy.exceptions import NotConfigured +from scrapy import signals from scrapy.utils.jsonrpc import jsonrpc_server_call from scrapy.utils.serialize import ScrapyJSONEncoder, ScrapyJSONDecoder from scrapy.utils.misc import load_object @@ -73,7 +75,7 @@ class WebService(server.Site): if not settings.getbool('WEBSERVICE_ENABLED'): raise NotConfigured logfile = settings['WEBSERVICE_LOGFILE'] - port = settings.getint('WEBSERVICE_PORT') + self.portnum = settings.getint('WEBSERVICE_PORT') root = RootResource() reslist = build_component_list(settings['WEBSERVICE_RESOURCES_BASE'], \ settings['WEBSERVICE_RESOURCES']) @@ -82,5 +84,12 @@ class WebService(server.Site): root.putChild(res.ws_name, res) server.Site.__init__(self, root, logPath=logfile) self.noisy = False - reactor.callWhenRunning(reactor.listenTCP, port, self) + dispatcher.connect(self.start_listening, signals.engine_started) + dispatcher.connect(self.stop_listening, signals.engine_stopped) + + def start_listening(self): + self.port = reactor.listenTCP(self.portnum, self) + + def stop_listening(self): + self.port.stopListening()