RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactoring of Crawler, Commands, Execution Queue and Spider Manager: 

Commands changes:

* removed (somewhat hacky) --init argument from settings command
* added set_crawler method to Commands, and a ``crawler`` property that returns
  a configured crawler. This way, commands that don't require a crawler (such
  as startproject) won't need to configure one.

Execution Queue changes:

* changed SERVICE_QUEUE_FILE setting to SQLITE_DB
* removed SERVICE_QUEUE setting
* added QUEUE_CLASS setting for defining the class to use for the execution queue
* added SERVER_QUEUE_CLASS setting for defining the class to use for the
  execution queue in server mode (runserver command)

Spider Manager changes:

* simplified SpiderManager API by removing the load() method
* added from_settings classmethod to SpiderManager
* added spider_modules constructor argument to SpiderManager

Crawler changes:

* added install() method to Crawler (to install it in scrapy.project) and
  uninstall() to remove it
* use CrawlerProcess.install() in scrapy.cmdline
* use crawler.install() and crawler.uninstall() in tests that a crawler in
  scrapy.project
* make telnet console and webservice play nicer with twisted by stopping
  listening when then engine goes down
* refactored Scrapy engine tests - it no longer uses the crawler singleton.
  Closes #215.
This commit is contained in:
Pablo Hoffman 2010-08-25 19:24:36 -03:00
parent eb51b9f785
commit 8fc78c4d0a
30 changed files with 231 additions and 239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/cmdline.py
View File

@ -7,6 +7,7 @@ import cProfile
import scrapy
from scrapy import log
from scrapy.crawler import CrawlerProcess
from scrapy.xlib import lsprofcalltree
from scrapy.conf import settings
from scrapy.command import ScrapyCommand
@ -79,6 +80,8 @@ def check_deprecated_scrapy_ctl(argv):
def execute(argv=None):
if argv is None:
argv = sys.argv
crawler = CrawlerProcess(settings)
crawler.install()
check_deprecated_scrapy_ctl(argv) # TODO: remove for Scrapy 0.11
cmds = _get_commands_dict()
cmdname = _get_command_name(argv)
@ -113,9 +116,8 @@ def execute(argv=None):
settings.defaults.update(cmd.default_settings)
del args[0] # remove command name from args
from scrapy.project import crawler
log.start()
crawler.configure()
cmd.set_crawler(crawler)
ret = _run_command(cmd, args, opts)
if ret is False:
parser.print_help()

8
scrapy/command.py
View File

@ -18,6 +18,14 @@ class ScrapyCommand(object):
# default settings to be used for this command instead of global defaults
default_settings = {}
def set_crawler(self, crawler):
self._crawler = crawler
@property
def crawler(self):
self._crawler.configure()
return self._crawler
def syntax(self):
"""
Command syntax (preferably one-line). Do not include command name.

15
scrapy/commands/crawl.py
View File

@ -1,10 +1,7 @@
from scrapy import log
from scrapy.command import ScrapyCommand
from scrapy.core.queue import ExecutionQueue
from scrapy.project import crawler
from scrapy.conf import settings
from scrapy.http import Request
from scrapy.project import crawler
from scrapy.utils.url import is_url
from collections import defaultdict
@ -32,31 +29,29 @@ class Command(ScrapyCommand):
settings.overrides['CRAWLSPIDER_FOLLOW_LINKS'] = False
def run(self, args, opts):
q = ExecutionQueue()
q = self.crawler.queue
urls, names = self._split_urls_and_names(args)
for name in names:
q.append_spider_name(name)
if opts.spider:
try:
spider = crawler.spiders.create(opts.spider)
spider = self.crawler.spiders.create(opts.spider)
for url in urls:
q.append_url(url, spider)
except KeyError:
log.msg('Unable to find spider: %s' % opts.spider, log.ERROR)
else:
for name, urls in self._group_urls_by_spider(urls):
spider = crawler.spiders.create(name)
spider = self.crawler.spiders.create(name)
for url in urls:
q.append_url(url, spider)
crawler.queue = q
crawler.start()
self.crawler.start()
def _group_urls_by_spider(self, urls):
spider_urls = defaultdict(list)
for url in urls:
spider_names = crawler.spiders.find_by_request(Request(url))
spider_names = self.crawler.spiders.find_by_request(Request(url))
if not spider_names:
log.msg('Could not find spider that handles url: %s' % url,
log.ERROR)

8
scrapy/commands/fetch.py
View File

@ -2,10 +2,8 @@ import pprint
from scrapy import log
from scrapy.command import ScrapyCommand
from scrapy.project import crawler
from scrapy.http import Request
from scrapy.spider import BaseSpider
from scrapy.project import crawler
from scrapy.utils.url import is_url
class Command(ScrapyCommand):
@ -44,11 +42,11 @@ class Command(ScrapyCommand):
spider = None
if opts.spider:
try:
spider = crawler.spiders.create(opts.spider)
spider = self.crawler.spiders.create(opts.spider)
except KeyError:
log.msg("Could not find spider: %s" % opts.spider, log.ERROR)
crawler.queue.append_request(request, spider, \
self.crawler.queue.append_request(request, spider, \
default_spider=BaseSpider('default'))
crawler.start()
self.crawler.start()

3
scrapy/commands/genspider.py
View File

@ -4,7 +4,6 @@ from os import listdir
from os.path import join, dirname, abspath, exists, splitext
import scrapy
from scrapy.project import crawler
from scrapy.command import ScrapyCommand
from scrapy.conf import settings
from scrapy.utils.template import render_templatefile, string_camelcase
@ -60,7 +59,7 @@ class Command(ScrapyCommand):
name, domain = args[0:2]
module = sanitize_module_name(name)
try:
spider = crawler.spiders.create(name)
spider = self.crawler.spiders.create(name)
except KeyError:
pass
else:

3
scrapy/commands/list.py
View File

@ -1,5 +1,4 @@
from scrapy.command import ScrapyCommand
from scrapy.project import crawler
class Command(ScrapyCommand):
@ -10,4 +9,4 @@ class Command(ScrapyCommand):
return "List available spiders"
def run(self, args, opts):
print "\n".join(crawler.spiders.list())
print "\n".join(self.crawler.spiders.list())

9
scrapy/commands/parse.py
View File

@ -1,5 +1,4 @@
from scrapy.command import ScrapyCommand
from scrapy.project import crawler
from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils import display
@ -69,11 +68,11 @@ class Command(ScrapyCommand):
def get_spider(self, request, opts):
if opts.spider:
try:
return crawler.spiders.create(opts.spider)
return self.crawler.spiders.create(opts.spider)
except KeyError:
log.msg('Unable to find spider: %s' % opts.spider, log.ERROR)
else:
spider = crawler.spiders.create_for_request(request)
spider = self.crawler.spiders.create_for_request(request)
if spider:
return spider
log.msg('Unable to find spider for: %s' % request, log.ERROR)
@ -84,8 +83,8 @@ class Command(ScrapyCommand):
spider = self.get_spider(request, opts)
if not spider:
return None, None
crawler.queue.append_request(request, spider)
crawler.start()
self.crawler.queue.append_request(request, spider)
self.crawler.start()
if not responses:
log.msg('No response downloaded for: %s' % request, log.ERROR, \
spider=spider)

5
scrapy/commands/queue.py
View File

@ -1,8 +1,9 @@
from scrapy.command import ScrapyCommand
from scrapy.commands import runserver
from scrapy.utils.misc import load_object
from scrapy.conf import settings
class Command(ScrapyCommand):
class Command(runserver.Command):
requires_project = True
default_settings = {'LOG_ENABLED': False}
@ -26,7 +27,7 @@ class Command(ScrapyCommand):
cmd = args[0]
botname = settings['BOT_NAME']
queue = load_object(settings['SERVICE_QUEUE'])().queue
queue = self.crawler.queue.queue
if cmd == 'add':
if len(args) < 2:

10
scrapy/commands/runserver.py
View File

@ -1,6 +1,4 @@
from scrapy.command import ScrapyCommand
from scrapy.project import crawler
from scrapy.utils.misc import load_object
from scrapy.conf import settings
class Command(ScrapyCommand):
@ -10,7 +8,9 @@ class Command(ScrapyCommand):
def short_desc(self):
return "Start Scrapy in server mode"
def process_options(self, args, opts):
super(Command, self).process_options(args, opts)
settings.overrides['QUEUE_CLASS'] = settings['SERVER_QUEUE_CLASS']
def run(self, args, opts):
queue_class = load_object(settings['SERVICE_QUEUE'])
crawler.queue = queue_class()
crawler.start()
self.crawler.start()

5
scrapy/commands/runspider.py
View File

@ -4,7 +4,6 @@ import os
from scrapy import log
from scrapy.utils.spider import iter_spider_classes
from scrapy.command import ScrapyCommand
from scrapy.project import crawler
def _import_file(filepath):
abspath = os.path.abspath(filepath)
@ -52,5 +51,5 @@ class Command(ScrapyCommand):
return
spider = spclasses.pop()()
# schedule spider and start engine
crawler.queue.append_spider(spider)
crawler.start()
self.crawler.queue.append_spider(spider)
self.crawler.start()

23
scrapy/commands/settings.py
View File

@ -1,5 +1,4 @@
from scrapy.command import ScrapyCommand
from scrapy.conf import settings as settings_
class Command(ScrapyCommand):
@ -24,26 +23,16 @@ class Command(ScrapyCommand):
help="print setting value, intepreted as an float")
parser.add_option("--getlist", dest="getlist", metavar="SETTING", \
help="print setting value, intepreted as an float")
parser.add_option("--init", dest="init", action="store_true", \
help="print initial setting value (before loading extensions and spiders)")
def process_options(self, args, opts):
super(Command, self).process_options(args, opts)
if opts.init:
self._print_setting(opts)
def run(self, args, opts):
if not opts.init:
self._print_setting(opts)
def _print_setting(self, opts):
settings = self.crawler.settings
if opts.get:
print settings_.get(opts.get)
print settings.get(opts.get)
elif opts.getbool:
print settings_.getbool(opts.getbool)
print settings.getbool(opts.getbool)
elif opts.getint:
print settings_.getint(opts.getint)
print settings.getint(opts.getint)
elif opts.getfloat:
print settings_.getfloat(opts.getfloat)
print settings.getfloat(opts.getfloat)
elif opts.getlist:
print settings_.getlist(opts.getlist)
print settings.getlist(opts.getlist)

10
scrapy/commands/shell.py
View File

@ -4,14 +4,13 @@ Scrapy Shell
See documentation in docs/topics/shell.rst
"""
from scrapy.project import crawler
from scrapy.core.queue import KeepAliveExecutionQueue
from scrapy.command import ScrapyCommand
from scrapy.shell import Shell
class Command(ScrapyCommand):
requires_project = False
default_settings = {'QUEUE_CLASS': 'scrapy.core.queue.KeepAliveExecutionQueue'}
def syntax(self):
return "[url|file]"
@ -30,7 +29,6 @@ class Command(ScrapyCommand):
def run(self, args, opts):
url = args[0] if args else None
shell = Shell(crawler, update_vars=self.update_vars, inthread=True)
shell.start(url=url).addBoth(lambda _: crawler.stop())
crawler.queue = KeepAliveExecutionQueue()
crawler.start()
shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True)
shell.start(url=url).addBoth(lambda _: self.crawler.stop())
self.crawler.start()

7
scrapy/conf/default_settings.py
View File

@ -175,6 +175,8 @@ MEMUSAGE_WARNING_MB = 0
NEWSPIDER_MODULE = ''
QUEUE_CLASS = 'scrapy.core.queue.ExecutionQueue'
RANDOMIZE_DOWNLOAD_DELAY = True
REDIRECT_MAX_METAREFRESH_DELAY = 100
@ -207,8 +209,7 @@ SCHEDULER_MIDDLEWARES_BASE = {
SCHEDULER_ORDER = 'DFO'
SERVICE_QUEUE = 'scrapy.contrib.queue.SqliteExecutionQueue'
SERVICE_QUEUE_FILE = 'scrapy.db'
SERVER_QUEUE_CLASS = 'scrapy.contrib.queue.SqliteExecutionQueue'
SPIDER_MANAGER_CLASS = 'scrapy.spidermanager.SpiderManager'
@ -227,6 +228,8 @@ SPIDER_MIDDLEWARES_BASE = {
SPIDER_MODULES = []
SQLITE_DB = 'scrapy.db'
SQS_QUEUE = 'scrapy'
SQS_VISIBILITY_TIMEOUT = 7200
SQS_POLLING_DELAY = 30

3
scrapy/contrib/pipeline/media.py
View File

@ -2,7 +2,6 @@ from twisted.internet.defer import Deferred, DeferredList
from scrapy.utils.defer import mustbe_deferred, defer_result
from scrapy import log
from scrapy.project import crawler
from scrapy.utils.request import request_fingerprint
from scrapy.utils.misc import arg_to_iter
@ -21,6 +20,8 @@ class MediaPipeline(object):
def __init__(self):
self.spiderinfo = {}
from scrapy.project import crawler
self.crawler = crawler
def open_spider(self, spider):
self.spiderinfo[spider] = self.SpiderInfo(spider)

2
scrapy/contrib/queue/__init__.py
View File

@ -8,7 +8,7 @@ class SqliteExecutionQueue(ExecutionQueue):
def __init__(self, *a, **kw):
super(SqliteExecutionQueue, self).__init__(*a, **kw)
self.queue = JsonSqlitePriorityQueue(settings['SERVICE_QUEUE_FILE'])
self.queue = JsonSqlitePriorityQueue(settings['SQLITE_DB'])
def _append_next(self):
msg = self.queue.pop()

2
scrapy/core/engine.py
View File

@ -40,7 +40,7 @@ class ExecutionEngine(object):
"""
self.scheduler = load_object(settings['SCHEDULER'])()
self.downloader = Downloader()
self.scraper = Scraper(self)
self.scraper = Scraper(self, self.crawler.settings)
self.configured = True
self._spider_closed_callback = spider_closed_callback

5
scrapy/core/queue.py
View File

@ -9,10 +9,7 @@ class ExecutionQueue(object):
polling_delay = 5
def __init__(self, _spiders=None):
if _spiders is None:
from scrapy.project import crawler
_spiders = crawler.spiders
def __init__(self, _spiders):
self.spider_requests = []
self._spiders = _spiders

3
scrapy/core/scraper.py
View File

@ -15,7 +15,6 @@ from scrapy.item import BaseItem
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy import log
from scrapy.stats import stats
from scrapy.conf import settings
class SpiderInfo(object):
@ -60,7 +59,7 @@ class SpiderInfo(object):
class Scraper(object):
def __init__(self, engine):
def __init__(self, engine, settings):
self.sites = {}
self.spidermw = SpiderMiddlewareManager()
itemproc_cls = load_object(settings['ITEM_PROCESSOR'])

29
scrapy/crawler.py
View File

@ -4,7 +4,6 @@ from twisted.internet import reactor, defer
from scrapy.xlib.pydispatch import dispatcher
from scrapy.core.engine import ExecutionEngine
from scrapy.core.queue import ExecutionQueue
from scrapy.extension import ExtensionManager
from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
from scrapy.utils.misc import load_object
@ -16,17 +15,28 @@ class Crawler(object):
def __init__(self, settings):
self.configured = False
self.settings = settings
self.spiders = load_object(settings['SPIDER_MANAGER_CLASS'])()
self.engine = ExecutionEngine(self)
def configure(self, queue=None):
self.extensions = ExtensionManager.from_settings(self.settings)
if not self.spiders.loaded:
self.spiders.load()
def install(self):
import scrapy.project
assert not hasattr(scrapy.project, 'crawler'), "crawler already installed"
scrapy.project.crawler = self
self.queue = queue or ExecutionQueue()
self.engine.configure(self._spider_closed)
def uninstall(self):
import scrapy.project
assert hasattr(scrapy.project, 'crawler'), "crawler not installed"
del scrapy.project.crawler
def configure(self):
if self.configured:
return
self.configured = True
self.engine = ExecutionEngine(self)
self.extensions = ExtensionManager.from_settings(self.settings)
spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
self.spiders = spman_cls.from_settings(self.settings)
queue_cls = load_object(self.settings['QUEUE_CLASS'])
self.queue = queue_cls(self.spiders)
self.engine.configure(self._spider_closed)
@defer.inlineCallbacks
def _start_next_spider(self):
@ -56,6 +66,7 @@ class Crawler(object):
@defer.inlineCallbacks
def start(self):
yield defer.maybeDeferred(self.configure)
yield defer.maybeDeferred(self.engine.start)
self._nextcall = reactor.callLater(0, self._start_next_spider)

13
scrapy/project.py
View File

@ -1,4 +1,11 @@
from scrapy.crawler import CrawlerProcess
from scrapy.conf import settings
"""
This module contains the Scrapy Crawler once installed by calling the crawler
``install`` method, like this::
crawler = CrawlerProcess(settings)
crawler.install()
After that, you can import the (singleton) crawler like this::
from scrapy.project import crawler
"""

32
scrapy/spidermanager.py
View File

@ -4,16 +4,26 @@ spiders
"""
from scrapy import log
from scrapy.conf import settings
from scrapy.utils.misc import walk_modules
from scrapy.utils.spider import iter_spider_classes
class SpiderManager(object):
def __init__(self):
self.loaded = False
def __init__(self, spider_modules):
self.spider_modules = spider_modules
self._spiders = {}
for name in self.spider_modules:
for module in walk_modules(name):
self._load_spiders(module)
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
self._spiders[spcls.name] = spcls
@classmethod
def from_settings(cls, settings):
return cls(settings.getlist('SPIDER_MODULES'))
def create(self, spider_name, **spider_kwargs):
"""Returns a Spider instance for the given spider name, using the given
@ -53,21 +63,5 @@ class SpiderManager(object):
"""Returns list of spiders available."""
return self._spiders.keys()
def load(self, spider_modules=None):
"""Load spiders from spider_modules or SPIDER_MODULES setting."""
if spider_modules is None:
spider_modules = settings.getlist('SPIDER_MODULES')
self.spider_modules = spider_modules
self._spiders = {}
for name in self.spider_modules:
for module in walk_modules(name):
self._load_spiders(module)
self.loaded = True
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
self._spiders[spcls.name] = spcls
def close_spider(self, spider):
pass

13
scrapy/telnet.py
View File

@ -10,9 +10,11 @@ from twisted.conch import manhole, telnet
from twisted.conch.insults import insults
from twisted.internet import reactor, protocol
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import NotConfigured
from scrapy.project import crawler
from scrapy.stats import stats
from scrapy import signals
from scrapy.utils.signal import send_catch_log
from scrapy.utils.trackref import print_live_refs
from scrapy.utils.engine import print_engine_status
@ -35,8 +37,15 @@ class TelnetConsole(protocol.ServerFactory):
if not settings.getbool('TELNETCONSOLE_ENABLED'):
raise NotConfigured
self.noisy = False
port = settings.getint('TELNETCONSOLE_PORT')
reactor.callWhenRunning(reactor.listenTCP, port, self)
self.portnum = settings.getint('TELNETCONSOLE_PORT')
dispatcher.connect(self.start_listening, signals.engine_started)
dispatcher.connect(self.stop_listening, signals.engine_stopped)
def start_listening(self):
self.port = reactor.listenTCP(self.portnum, self)
def stop_listening(self):
self.port.stopListening()
def protocol(self):
telnet_vars = self._get_telnet_vars()

9
scrapy/tests/test_cmdline/__init__.py
View File

@ -20,29 +20,20 @@ class CmdlineTest(unittest.TestCase):
return comm[0].strip()
def test_default_settings(self):
self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init'), \
'default')
self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
'default + loaded + started')
def test_override_settings_using_settings_arg(self):
self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init', \
'--settings', 'scrapy.tests.test_cmdline.settings2'), \
'override')
self.assertEqual(self._execute('settings', '--get', 'TEST1', \
'--settings', 'scrapy.tests.test_cmdline.settings2'), \
'override + loaded + started')
def test_override_settings_using_set_arg(self):
self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init', '--set', 'TEST1=override'), \
'override')
self.assertEqual(self._execute('settings', '--get', 'TEST1', '--set', 'TEST1=override'), \
'override + loaded + started')
def test_override_settings_using_envvar(self):
self.env['SCRAPY_TEST1'] = 'override'
self.assertEqual(self._execute('settings', '--get', 'TEST1', '--init'), \
'override')
self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
'override + loaded + started')

184
scrapy/tests/test_engine.py
View File

@ -1,20 +1,19 @@
"""
Scrapy engine tests
"""
import sys, os, re, urlparse
import sys, os, re, urlparse, unittest
from twisted.internet import reactor
from twisted.web import server, resource, static, util
from twisted.internet import reactor, defer
from twisted.web import server, static, util
from twisted.trial import unittest
from scrapy import signals
from scrapy.project import crawler
from scrapy.conf import Settings
from scrapy.crawler import Crawler
from scrapy.xlib.pydispatch import dispatcher
from scrapy.tests import tests_datadir
from scrapy.spider import BaseSpider
from scrapy.item import Item, Field
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.http import Request
from scrapy.utils.signal import disconnect_all
class TestItem(Item):
name = Field()
@ -24,7 +23,6 @@ class TestItem(Item):
class TestSpider(BaseSpider):
name = "scrapytest.org"
allowed_domains = ["scrapytest.org", "localhost"]
start_urls = ['http://localhost']
itemurl_re = re.compile("item\d+.html")
name_re = re.compile("<h1>(.*?)</h1>", re.M)
@ -48,66 +46,61 @@ class TestSpider(BaseSpider):
item['price'] = m.group(1)
return item
#class TestResource(resource.Resource):
# isLeaf = True
#
# def render_GET(self, request):
# return "hello world!"
def start_test_site():
def start_test_site(debug=False):
root_dir = os.path.join(tests_datadir, "test_site")
r = static.File(root_dir)
# r.putChild("test", TestResource())
r.putChild("redirect", util.Redirect("/redirected"))
r.putChild("redirected", static.Data("Redirected here", "text/plain"))
port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
if debug:
print "Test server running at http://localhost:%d/ - hit Ctrl-C to finish." \
% port.getHost().port
return port
class CrawlingSession(object):
class CrawlerRun(object):
"""A class to run the crawler and keep track of events occurred"""
def __init__(self):
self.name = 'scrapytest.org'
self.spider = None
self.respplug = []
self.reqplug = []
self.itemresp = []
self.signals_catched = {}
self.wasrun = False
def run(self):
self.port = start_test_site()
self.portno = self.port.getHost().port
self.spider = TestSpider()
if self.spider:
self.spider.start_urls = [
self.geturl("/"),
self.geturl("/redirect"),
]
start_urls = [self.geturl("/"), self.geturl("/redirect")]
self.spider = TestSpider(start_urls=start_urls)
dispatcher.connect(self.record_signal, signals.engine_started)
dispatcher.connect(self.record_signal, signals.engine_stopped)
dispatcher.connect(self.record_signal, signals.spider_opened)
dispatcher.connect(self.record_signal, signals.spider_idle)
dispatcher.connect(self.record_signal, signals.spider_closed)
dispatcher.connect(self.item_scraped, signals.item_scraped)
dispatcher.connect(self.request_received, signals.request_received)
dispatcher.connect(self.response_downloaded, signals.response_downloaded)
for name, signal in vars(signals).items():
if not name.startswith('_'):
dispatcher.connect(self.record_signal, signal)
dispatcher.connect(self.item_scraped, signals.item_scraped)
dispatcher.connect(self.request_received, signals.request_received)
dispatcher.connect(self.response_downloaded, signals.response_downloaded)
crawler.configure()
crawler.queue.append_spider(self.spider)
crawler.start()
self.port.stopListening()
self.wasrun = True
# FIXME: extremly ugly hack to avoid propagating errors to other
# stats because of living signals. This whole test_engine.py should
# be rewritten from scratch actually.
from scrapy.utils.signal import disconnect_all
disconnect_all(signals.stats_spider_opened)
disconnect_all(signals.stats_spider_closing)
disconnect_all(signals.stats_spider_closed)
settings = Settings()
self.crawler = Crawler(settings)
self.crawler.install()
self.crawler.configure()
self.crawler.queue.append_spider(self.spider)
self.crawler.start()
self.deferred = defer.Deferred()
dispatcher.connect(self.stop, signals.engine_stopped)
return self.deferred
def stop(self):
self.port.stopListening()
for name, signal in vars(signals).items():
if not name.startswith('_'):
disconnect_all(signal)
self.crawler.uninstall()
self.deferred.callback(None)
def geturl(self, path):
return "http://localhost:%s%s" % (self.portno, path)
@ -132,67 +125,49 @@ class CrawlingSession(object):
signalargs.pop('sender', None)
self.signals_catched[sig] = signalargs
session = CrawlingSession()
class EngineTest(unittest.TestCase):
def setUp(self):
if not session.wasrun:
session.run()
@defer.inlineCallbacks
def test_crawler(self):
self.run = CrawlerRun()
yield self.run.run()
self._assert_visited_urls()
self._assert_received_requests()
self._assert_downloaded_responses()
self._assert_scraped_items()
self._assert_signals_catched()
def test_spider_locator(self):
"""
Check the spider is loaded and located properly via the SpiderLocator
"""
assert session.spider is not None
self.assertEqual(session.spider.name, session.name)
def test_visited_urls(self):
"""
Make sure certain URls were actually visited
"""
# expected urls that should be visited
def _assert_visited_urls(self):
must_be_visited = ["/", "/redirect", "/redirected",
"/item1.html", "/item2.html", "/item999.html"]
urls_visited = set([rp[0].url for rp in session.respplug])
urls_expected = set([session.geturl(p) for p in must_be_visited])
urls_visited = set([rp[0].url for rp in self.run.respplug])
urls_expected = set([self.run.geturl(p) for p in must_be_visited])
assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited)
def test_requests_received(self):
"""
Check requests received
"""
def _assert_received_requests(self):
# 3 requests should be received from the spider. start_urls and redirects don't count
self.assertEqual(3, len(session.reqplug))
self.assertEqual(3, len(self.run.reqplug))
paths_expected = ['/item999.html', '/item2.html', '/item1.html']
urls_requested = set([rq[0].url for rq in session.reqplug])
urls_expected = set([session.geturl(p) for p in paths_expected])
urls_requested = set([rq[0].url for rq in self.run.reqplug])
urls_expected = set([self.run.geturl(p) for p in paths_expected])
assert urls_expected <= urls_requested
def test_responses_downloaded(self):
"""
Check responses downloaded
"""
def _assert_downloaded_responses(self):
# response tests
self.assertEqual(6, len(session.respplug))
self.assertEqual(6, len(self.run.respplug))
for response, spider in session.respplug:
if session.getpath(response.url) == '/item999.html':
for response, _ in self.run.respplug:
if self.run.getpath(response.url) == '/item999.html':
self.assertEqual(404, response.status)
if session.getpath(response.url) == '/redirect':
if self.run.getpath(response.url) == '/redirect':
self.assertEqual(302, response.status)
def test_item_data(self):
"""
Check item data
"""
# item tests
self.assertEqual(2, len(session.itemresp))
for item, response in session.itemresp:
def _assert_scraped_items(self):
self.assertEqual(2, len(self.run.itemresp))
for item, response in self.run.itemresp:
self.assertEqual(item['url'], response.url)
if 'item1.html' in item['url']:
self.assertEqual('Item 1 name', item['name'])
@ -201,29 +176,22 @@ class EngineTest(unittest.TestCase):
self.assertEqual('Item 2 name', item['name'])
self.assertEqual('200', item['price'])
def test_signals(self):
"""
Check signals were sent properly
"""
from scrapy import signals
def _assert_signals_catched(self):
assert signals.engine_started in self.run.signals_catched
assert signals.engine_stopped in self.run.signals_catched
assert signals.spider_opened in self.run.signals_catched
assert signals.spider_idle in self.run.signals_catched
assert signals.spider_closed in self.run.signals_catched
assert signals.engine_started in session.signals_catched
assert signals.engine_stopped in session.signals_catched
assert signals.spider_opened in session.signals_catched
assert signals.spider_idle in session.signals_catched
assert signals.spider_closed in session.signals_catched
self.assertEqual({'spider': self.run.spider},
self.run.signals_catched[signals.spider_opened])
self.assertEqual({'spider': self.run.spider},
self.run.signals_catched[signals.spider_idle])
self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
self.run.signals_catched[signals.spider_closed])
self.assertEqual({'spider': session.spider},
session.signals_catched[signals.spider_opened])
self.assertEqual({'spider': session.spider},
session.signals_catched[signals.spider_idle])
self.assertEqual({'spider': session.spider, 'reason': 'finished'},
session.signals_catched[signals.spider_closed])
if __name__ == "__main__":
if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
port = start_test_site()
print "Test server running at http://localhost:%d/ - hit Ctrl-C to finish." % port.getHost().port
start_test_site(debug=True)
reactor.run()
else:
unittest.main()

4
scrapy/tests/test_pipeline_images.py
View File

@ -1,5 +1,6 @@
import os
from twisted.trial import unittest
from scrapy.crawler import Crawler
from scrapy.conf import settings
from tempfile import mkdtemp
from shutil import rmtree
@ -7,6 +8,8 @@ from shutil import rmtree
class ImagesPipelineTestCase(unittest.TestCase):
def setUp(self):
self.crawler = Crawler(settings)
self.crawler.install()
try:
import Image
except ImportError, e:
@ -23,6 +26,7 @@ class ImagesPipelineTestCase(unittest.TestCase):
del self.pipeline
rmtree(self.tempdir)
settings.disabled = self.settings_disabled_before
self.crawler.uninstall()
def test_image_path(self):
image_path = self.pipeline.image_key

5
scrapy/tests/test_pipeline_media.py
View File

@ -2,6 +2,8 @@ from twisted.trial import unittest
from twisted.python import failure
from twisted.internet import defer, reactor
from scrapy.conf import Settings
from scrapy.crawler import Crawler
from scrapy.http import Request, Response
from scrapy.spider import BaseSpider
from scrapy.utils.request import request_fingerprint
@ -29,12 +31,15 @@ class MediaPipelineTestCase(unittest.TestCase):
pipeline_class = _MockedMediaPipeline
def setUp(self):
self.crawler = Crawler(Settings())
self.crawler.install()
self.spider = BaseSpider('media.com')
self.pipe = self.pipeline_class()
self.pipe.open_spider(self.spider)
def tearDown(self):
self.pipe.close_spider(self.spider)
self.crawler.uninstall()
@defer.inlineCallbacks
def test_return_item_by_default(self):

9
scrapy/tests/test_spidermanager/__init__.py
View File

@ -22,10 +22,7 @@ class SpiderManagerTest(unittest.TestCase):
self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
shutil.copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir)
self.spiderman = SpiderManager()
assert not self.spiderman.loaded
self.spiderman.load(['test_spiders_xxx'])
assert self.spiderman.loaded
self.spiderman = SpiderManager(['test_spiders_xxx'])
def tearDown(self):
del self.spiderman
@ -57,9 +54,9 @@ class SpiderManagerTest(unittest.TestCase):
['spider3'])
def test_load_spider_module(self):
self.spiderman.load(['scrapy.tests.test_spidermanager.test_spiders.spider1'])
self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider1'])
assert len(self.spiderman._spiders) == 1
def test_load_base_spider(self):
self.spiderman.load(['scrapy.tests.test_spidermanager.test_spiders.spider0'])
self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0'])
assert len(self.spiderman._spiders) == 0

22
scrapy/tests/test_utils_jsonrpc.py
View File

@ -3,6 +3,8 @@ from cStringIO import StringIO
from scrapy.utils.jsonrpc import jsonrpc_client_call, jsonrpc_server_call, \
JsonRpcError, jsonrpc_errors
from scrapy.utils.serialize import ScrapyJSONDecoder
from scrapy.tests.test_utils_serialize import ExecutionMangerStub
from scrapy.utils.py26 import json
class urllib_stub(object):
@ -30,6 +32,10 @@ class TestTarget(object):
class JsonRpcUtilsTestCase(unittest.TestCase):
def setUp(self):
crawler = ExecutionMangerStub([])
self.json_decoder = ScrapyJSONDecoder(crawler=crawler)
def test_jsonrpc_client_call_request(self):
ul = urllib_stub(1)
jsonrpc_client_call('url', 'test', 'one', 2, _urllib=ul)
@ -62,26 +68,26 @@ class JsonRpcUtilsTestCase(unittest.TestCase):
def test_jsonrpc_server_call(self):
t = TestTarget()
r = jsonrpc_server_call(t, 'invalid json data')
r = jsonrpc_server_call(t, 'invalid json data', self.json_decoder)
assert 'error' in r
assert r['jsonrpc'] == '2.0'
assert r['id'] is None
self.assertEqual(r['error']['code'], jsonrpc_errors.PARSE_ERROR)
assert 'Traceback' in r['error']['data']
r = jsonrpc_server_call(t, '{"test": "test"}')
r = jsonrpc_server_call(t, '{"test": "test"}', self.json_decoder)
assert 'error' in r
assert r['jsonrpc'] == '2.0'
assert r['id'] is None
self.assertEqual(r['error']['code'], jsonrpc_errors.INVALID_REQUEST)
r = jsonrpc_server_call(t, '{"method": "notfound", "id": 1}')
r = jsonrpc_server_call(t, '{"method": "notfound", "id": 1}', self.json_decoder)
assert 'error' in r
assert r['jsonrpc'] == '2.0'
assert r['id'] == 1
self.assertEqual(r['error']['code'], jsonrpc_errors.METHOD_NOT_FOUND)
r = jsonrpc_server_call(t, '{"method": "exception", "id": 1}')
r = jsonrpc_server_call(t, '{"method": "exception", "id": 1}', self.json_decoder)
assert 'error' in r
assert r['jsonrpc'] == '2.0'
assert r['id'] == 1
@ -89,19 +95,21 @@ class JsonRpcUtilsTestCase(unittest.TestCase):
assert 'testing-errors' in r['error']['message']
assert 'Traceback' in r['error']['data']
r = jsonrpc_server_call(t, '{"method": "call", "id": 2}')
r = jsonrpc_server_call(t, '{"method": "call", "id": 2}', self.json_decoder)
assert 'result' in r
assert r['jsonrpc'] == '2.0'
assert r['id'] == 2
self.assertEqual(r['result'], ([], {}))
r = jsonrpc_server_call(t, '{"method": "call", "params": [456, 123], "id": 3}')
r = jsonrpc_server_call(t, '{"method": "call", "params": [456, 123], "id": 3}', \
self.json_decoder)
assert 'result' in r
assert r['jsonrpc'] == '2.0'
assert r['id'] == 3
self.assertEqual(r['result'], ([456, 123], {}))
r = jsonrpc_server_call(t, '{"method": "call", "params": {"data": 789}, "id": 3}')
r = jsonrpc_server_call(t, '{"method": "call", "params": {"data": 789}, "id": 3}', \
self.json_decoder)
assert 'result' in r
assert r['jsonrpc'] == '2.0'
assert r['id'] == 3

10
scrapy/utils/serialize.py
View File

@ -4,7 +4,6 @@ import decimal
from twisted.internet import defer
from scrapy.project import crawler
from scrapy.spider import BaseSpider
from scrapy.http import Request, Response
from scrapy.utils.py26 import json
@ -20,8 +19,10 @@ class SpiderReferencer(object):
spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$')
def __init__(self, manager=None):
self.manager = manager or crawler
def __init__(self, crawler=None):
if crawler is None:
from scrapy.project import crawler
self.manager = crawler
def get_reference_from_spider(self, spider):
return 'spider:%x:%s' % (id(spider), spider.name)
@ -109,7 +110,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
class ScrapyJSONDecoder(json.JSONDecoder):
def __init__(self, *a, **kw):
self.spref = kw.pop('spref', None) or SpiderReferencer()
crawler = kw.pop('crawler', None)
self.spref = kw.pop('spref', None) or SpiderReferencer(crawler)
super(ScrapyJSONDecoder, self).__init__(*a, **kw)
def decode(self, s):

13
scrapy/webservice.py
View File

@ -7,7 +7,9 @@ See docs/topics/ws.rst
from twisted.internet import reactor
from twisted.web import server, resource, error
from scrapy.xlib.pydispatch import dispatcher
from scrapy.exceptions import NotConfigured
from scrapy import signals
from scrapy.utils.jsonrpc import jsonrpc_server_call
from scrapy.utils.serialize import ScrapyJSONEncoder, ScrapyJSONDecoder
from scrapy.utils.misc import load_object
@ -73,7 +75,7 @@ class WebService(server.Site):
if not settings.getbool('WEBSERVICE_ENABLED'):
raise NotConfigured
logfile = settings['WEBSERVICE_LOGFILE']
port = settings.getint('WEBSERVICE_PORT')
self.portnum = settings.getint('WEBSERVICE_PORT')
root = RootResource()
reslist = build_component_list(settings['WEBSERVICE_RESOURCES_BASE'], \
settings['WEBSERVICE_RESOURCES'])
@ -82,5 +84,12 @@ class WebService(server.Site):
root.putChild(res.ws_name, res)
server.Site.__init__(self, root, logPath=logfile)
self.noisy = False
reactor.callWhenRunning(reactor.listenTCP, port, self)
dispatcher.connect(self.start_listening, signals.engine_started)
dispatcher.connect(self.stop_listening, signals.engine_stopped)
def start_listening(self):
self.port = reactor.listenTCP(self.portnum, self)
def stop_listening(self):
self.port.stopListening()