diff --git a/scrapy/trunk/INSTALL b/scrapy/trunk/INSTALL new file mode 100644 index 000000000..ea6b39db2 --- /dev/null +++ b/scrapy/trunk/INSTALL @@ -0,0 +1,31 @@ +You need: + * Python 2.5 + * Twisted + * libxml2 python bindings. version 2.6.28 or above is highly recommended. + * pyopenssl for HTTPS support + * for win32: http://webcleaner.sourceforge.net/pyOpenSSL-0.6.win32-py2.5.exe + * spidermonkey (optional) - required for JSParser + +The python interpreter should have sys.getdefaultencoding() == 'utf-8' +If this is not the case, you can add the following lines to a file +[PYTHON_INSTALL]/site-packages/sitecustomize.py: +import sys +sys.setdefaultencoding('utf-8') + + +INSTALLING LIBRARIES +-------------------- + +The procedure for installing the required third party libraries (twisted, +libxml2 and pyopenssl) depends on the platform and OS you use. + +Visit the project homepages for more information about it: + + * Twisted: http://twistedmatrix.com/ + * libxml2: http://xmlsoft.org/ + * releases available here: ftp://xmlsoft.org/libxml2/ + * pyopenssl: http://pyopenssl.sourceforge.net/ + + +In Debian/Ubuntu Linux it's as easy as: +apt-get install python-twisted python-libxml2 python-pyopenssl diff --git a/scrapy/trunk/README b/scrapy/trunk/README new file mode 100644 index 000000000..e59d381f7 --- /dev/null +++ b/scrapy/trunk/README @@ -0,0 +1,3 @@ +This is Scrapy, an opensource screen scraping framework written in Python. + +For more visit the project home page at http://scrapy.org diff --git a/scrapy/trunk/extras/sql/scraping.sql b/scrapy/trunk/extras/sql/scraping.sql new file mode 100644 index 000000000..c3314f5a1 --- /dev/null +++ b/scrapy/trunk/extras/sql/scraping.sql @@ -0,0 +1,72 @@ +DROP TABLE IF EXISTS `url_history`; +DROP TABLE IF EXISTS `version`; +DROP TABLE IF EXISTS `url_status`; +DROP TABLE IF EXISTS `ticket`; +DROP TABLE IF EXISTS `domain_stats`; +DROP TABLE IF EXISTS `domain_stats_history`; +DROP TABLE IF EXISTS `domain_data_history`; + +CREATE TABLE `ticket` ( + `guid` char(40) NOT NULL, + `domain` varchar(255) default NULL, + `url` varchar(2048) default NULL, + `url_hash` char(40) default NULL, -- so we can join to url_status + PRIMARY KEY (`guid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `version` ( + `id` bigint(20) NOT NULL auto_increment, + `guid` char(40) NOT NULL, + `version` char(40) NOT NULL, + `seen` datetime NOT NULL, + PRIMARY KEY (`id`), + FOREIGN KEY (`guid`) REFERENCES ticket(guid) ON UPDATE CASCADE ON DELETE CASCADE, + UNIQUE KEY (`version`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `url_status` ( + -- see http://support.microsoft.com/kb/q208427/ for explanation of 2048 + `url_hash` char(40) NOT NULL, -- for faster searches + `url` varchar(2048) NOT NULL, + `parent_hash` char(40) default NULL, -- the url that was followed to this one - for reporting + `last_version` char(40) default NULL, -- can be null if it generated an error the last time is was checked + `last_checked` datetime NOT NULL, + PRIMARY KEY (`url_hash`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `url_history` ( + `url_hash` char(40) NOT NULL, + `version` char(40) NOT NULL, + `postdata_hash` char(40) default NULL, + `created` datetime NOT NULL, + PRIMARY KEY (`version`), + FOREIGN KEY (`url_hash`) REFERENCES url_status(url_hash) ON UPDATE CASCADE ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `domain_stats` ( + `key1` varchar(128) NOT NULL, + `key2` varchar(128) NOT NULL, + `value` text, + PRIMARY KEY `key1_key2` (`key1`, `key2`), + KEY `key1` (`key1`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `domain_stats_history` ( + `id` bigint(20) NOT NULL auto_increment, + `key1` varchar(128) NOT NULL, + `key2` varchar(128) NOT NULL, + `value` varchar(2048) NOT NULL, + `stored` datetime NOT NULL, + PRIMARY KEY (`id`), + KEY `key1_key2` (`key1`, `key2`), + KEY `key1` (`key1`), + KEY `stored` (`stored`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `domain_data_history` ( + `domain` varchar(255) NOT NULL, + `stored` datetime NOT NULL, + `data` text, + KEY `domain_stored` (`domain`, `stored`), + KEY `domain` (`domain`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/scrapy/trunk/scrapy/__init__.py b/scrapy/trunk/scrapy/__init__.py new file mode 100644 index 000000000..9b40defb8 --- /dev/null +++ b/scrapy/trunk/scrapy/__init__.py @@ -0,0 +1,18 @@ +""" +Scrapy - a screen scraping framework written in Python +""" + +__version__ = "0.1.0" + +import sys, os + +if sys.version_info < (2,5): + print "Scrapy %s requires Python 2.5 or above" % __version__ + sys.exit(1) + +# add external python libraries bundled into scrapy +sys.path.insert(0, os.path.join(os.path.abspath(os.path.dirname(__file__)), "lib")) + +# monkey patches to fix external library issues +from scrapy.patches import monkeypatches +monkeypatches.apply_patches() diff --git a/scrapy/trunk/scrapy/command/__init__.py b/scrapy/trunk/scrapy/command/__init__.py new file mode 100644 index 000000000..0692578bb --- /dev/null +++ b/scrapy/trunk/scrapy/command/__init__.py @@ -0,0 +1 @@ +from scrapy.command.models import ScrapyCommand diff --git a/scrapy/trunk/scrapy/command/cmdline.py b/scrapy/trunk/scrapy/command/cmdline.py new file mode 100644 index 000000000..e9f3e52fc --- /dev/null +++ b/scrapy/trunk/scrapy/command/cmdline.py @@ -0,0 +1,154 @@ +from __future__ import with_statement + +import sys +import os +import optparse + +import scrapy +from scrapy.core import log +from scrapy.spider import spiders +from scrapy.conf import settings + +def find_commands(dir): + try: + return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and f.endswith('.py')] + except OSError: + return [] + +def builtin_commands_dict(): + d = {} + scrapy_dir = scrapy.__path__[0] + commands_dir = os.path.join(scrapy_dir, 'command', 'commands') + for cmdname in find_commands(commands_dir): + modname = 'scrapy.command.commands.%s' % cmdname + command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) + if callable(command): + d[cmdname] = command() + else: + print 'WARNING: Builtin command module %s exists but Command class not found' % modname + return d + +def custom_commands_dict(): + d = {} + cmdsmod = settings['COMMANDS_MODULE'] + if cmdsmod: + mod = __import__(cmdsmod, {}, {}, ['']) + for cmdname in find_commands(mod.__path__[0]): + modname = '%s.%s' % (cmdsmod, cmdname) + command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) + if callable(command): + d[cmdname] = command() + else: + print 'WARNING: Custom command module %s exists but Command class not found' % modname + return d + +def getcmdname(): + for arg in sys.argv[1:]: + if not arg.startswith('-'): + return arg + +def usage(): + s = "usage: %s [options] [args]\n" % sys.argv[0] + s += " %s -h\n\n" % sys.argv[0] + s += "Built-in subcommands:\n" + + builtin_cmds = builtin_commands_dict() + custom_cmds = custom_commands_dict() + + filtered_builtin_cmds = [(name, cls) for name, cls in builtin_cmds.iteritems() if name not in custom_cmds] + + for cmdname, cmdclass in filtered_builtin_cmds: + s += " %s %s\n" % (cmdname, cmdclass.syntax()) + s += " %s\n" % cmdclass.short_desc() + + if custom_cmds: + s += "\n" + s += "Custom (or overloaded) subcommands:\n" + for cmdname, cmdclass in custom_cmds.iteritems(): + s += " %s %s\n" % (cmdname, cmdclass.syntax()) + s += " %s\n" % cmdclass.short_desc() + + return s + + +def update_defaults(defaults, module): + settingsdict = vars(module) + for k, v in settingsdict.iteritems(): + if not k.startswith("_"): + defaults[k] = v + +def command_settings(cmdname): + try: + module = __import__('%s.%s' % ('scrapy.conf.commands', cmdname), {}, {}, ['']) + update_defaults(settings.defaults, module) + except ImportError: + pass + + basepath = settings['COMMANDS_SETTINGS_MODULE'] + if basepath: + try: + module = __import__('%s.%s' % (basepath, cmdname), {}, {}, ['']) + update_defaults(settings.defaults, module) + except ImportError: + pass + +# This dict holds information about the executed command for later use +command_executed = {} + +def execute(): + spiders.load() + cmds = builtin_commands_dict() + cmds.update(custom_commands_dict()) + + cmdname = getcmdname() + command_settings(cmdname) + + if not cmdname: + print "Scrapy %s\n" % scrapy.__version__ + print usage() + sys.exit() + + parser = optparse.OptionParser() + + if cmdname in cmds: + cmd = cmds[cmdname] + cmd.add_options(parser) + parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) + parser.description = cmd.long_desc() + else: + print "Scrapy %s\n" % scrapy.__version__ + print "Unknown command: %s\n" % cmdname + print 'Type "%s -h" for help' % sys.argv[0] + sys.exit() + + (opts, args) = parser.parse_args() + del args[0] # args[0] is cmdname + + # storing command executed info for later reference + command_executed['name'] = cmdname + command_executed['class'] = cmd + command_executed['args'] = args[:] + command_executed['opts'] = opts.__dict__.copy() + + cmd.process_options(args, opts) + log.start() # start logging + if opts.profile: + log.msg("Profiling enabled. Analyze later with: python -m pstats %s" % opts.profile) + import cProfile + loc = locals() + p = cProfile.Profile() + p.runctx('ret = cmd.run(args, opts)', globals(), loc) + p.dump_stats(opts.profile) + try: + import lsprofcalltree + fn = opts.profile + ".cachegrind" + k = lsprofcalltree.KCacheGrind(p) + with open(fn, 'w') as f: + k.output(f) + except ImportError: + pass + ret = loc['ret'] + else: + ret = cmd.run(args, opts) + if ret is False: + parser.print_help() diff --git a/scrapy/trunk/scrapy/command/commands/__init__.py b/scrapy/trunk/scrapy/command/commands/__init__.py new file mode 100644 index 000000000..216e4e89a --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/__init__.py @@ -0,0 +1,67 @@ +""" +Commmands for controlling Scrapy +""" +from scrapy.conf import settings + +class ScrapyCommand(object): + def syntax(self): + """ + Command syntax (preferably one-line). Do not include command name. + """ + return "" + + def short_desc(self): + """ + A short description of the command + """ + return "" + + def long_desc(self): + """ + A long description of the command. Return short description when not + available. It cannot contain newlines, since contents will be formatted + by optparser which removes newlines and wraps text. + """ + return self.short_desc() + + def help(self): + """ + An extensive help for the command. It will be shown when using the + "help" command. It can contain newlines, since not post-formatting will + be applied to its contents. + """ + return self.long_desc() + + def add_options(self, parser): + """ + Populate option parse with options available for this command + """ + parser.add_option("-f", "--logfile", dest="logfile", help="logfile to use. if omitted stderr will be used", metavar="FILE") + parser.add_option("-o", "--loglevel", dest="loglevel", default=None, help="log level") + parser.add_option("--spider", dest="spider", default=None, help="default spider (domain) to use if no spider is found") + parser.add_option("--nolog", dest="nolog", action="store_true", help="disable all log messages") + parser.add_option("--profile", dest="profile", default=None, help="write profiling stats in FILE, to analyze later with: python -m pstats FILE", metavar="FILE") + + def process_options(self, args, opts): + if opts.logfile: + settings.overrides['LOG_ENABLED'] = True + settings.overrides['LOGFILE'] = opts.logfile + + if opts.loglevel: + settings.overrides['LOG_ENABLED'] = True + settings.overrides['LOGLEVEL'] = opts.loglevel + + if opts.nolog: + settings.overrides['LOG_ENABLED'] = False + + if opts.spider: + from scrapy.spider import spiders + spiders.default_domain = opts.spider + + def run(self, args, opts): + """ + Entry point for running commands + """ + raise NotImplementedError + + diff --git a/scrapy/trunk/scrapy/command/commands/crawl.py b/scrapy/trunk/scrapy/command/commands/crawl.py new file mode 100644 index 000000000..cb176d2b2 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/crawl.py @@ -0,0 +1,38 @@ +from scrapy.command import ScrapyCommand +from scrapy.core.manager import scrapymanager +from scrapy.replay import Replay +from scrapy.conf import settings + + +class Command(ScrapyCommand): + def syntax(self): + return "[options] [domain|url] ..." + + def short_desc(self): + return "Run the web scraping engine from the command line" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("--restrict", dest="restrict", action="store_true", help="restrict crawling only to the given urls") + parser.add_option("--record", dest="record", help="use FILE for recording session (see replay command)", metavar="FILE") + parser.add_option("--record-dir", dest="recorddir", help="use DIR for recording (instead of file)", metavar="DIR") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + if opts.nopipeline: + settings.overrides['ITEM_PIPELINES'] = [] + + if opts.nocache: + settings.overrides['CACHE2_DIR'] = None + + if opts.restrict: + settings.overrides['RESTRICT_TO_URLS'] = args + + if opts.record or opts.recorddir: + # self.replay is used for preventing Replay signals handler from + # disconnecting since pydispatcher uses weak references + self.replay = Replay(opts.record or opts.recorddir, mode='record', usedir=bool(opts.recorddir)) + self.replay.record(args=args, opts=opts.__dict__) + + def run(self, args, opts): + scrapymanager.runonce(*args, **opts.__dict__) diff --git a/scrapy/trunk/scrapy/command/commands/download.py b/scrapy/trunk/scrapy/command/commands/download.py new file mode 100644 index 000000000..a8a09a0f0 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/download.py @@ -0,0 +1,28 @@ +import sys, pprint + +from scrapy.command import ScrapyCommand +from scrapy.fetcher import fetch + +class Command(ScrapyCommand): + def syntax(self): + return "[options] " + + def short_desc(self): + return "Download a URL using the Scrapy downloader" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-s", "--source", dest="source", action="store_true", help="output HTTP body only") + parser.add_option("--headers", dest="headers", action="store_true", help="output HTTP headers only") + + def run(self, args, opts): + if not args: + print "A URL is required" + return + + responses = fetch(args) + if responses: + if opts.headers: + pprint.pprint(responses[0].headers) + else: + sys.stdout.write(str(responses[0].body)) diff --git a/scrapy/trunk/scrapy/command/commands/genspider.py b/scrapy/trunk/scrapy/command/commands/genspider.py new file mode 100644 index 000000000..bb72916ef --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/genspider.py @@ -0,0 +1,52 @@ +import os + +from scrapy.spider import spiders +from scrapy.command import ScrapyCommand +from scrapy.conf import settings + +class Command(ScrapyCommand): + def syntax(self): + return " " + + def short_desc(self): + return "Generate new spider based on predefined template" + + def run(self, args, opts): + if len(args) != 2: + return False + + name = args[0] + site = args[1] + spiders_dict = spiders.asdict() + if not name in spiders_dict.keys(): + self._genspider(name, site) + else: + print "Spider '%s' exist" % name + + def _genspider(self, name, site): + """ Generate spider """ + tvars = { + 'name': name, + 'site': site, + 'classname': '%sSpider' % ''.join([s.capitalize() for s in name.split('-')]) + } + + spiders_module = __import__(settings['NEWSPIDER_MODULE'], {}, {}, ['']) + spidersdir = os.path.abspath(os.path.dirname(spiders_module.__file__)) + self._genfiles('spider.tmpl', '%s/%s.py' % (spidersdir, name), tvars) + + def _genfiles(self, template_name, source_name, tvars): + """ Generate source from template, substitute variables """ + template_file = os.path.join(settings['TEMPLATES_DIR'], template_name) + tmpl = open(template_file) + clines = [] + for l in tmpl.readlines(): + for key, val in tvars.items(): + l = l.replace('@%s@' % key, val) + clines.append(l) + tmpl.close() + source = ''.join(clines) + if not os.path.exists(source_name): + sfile = open(source_name, "w") + sfile.write(source) + sfile.close() diff --git a/scrapy/trunk/scrapy/command/commands/getattr.py b/scrapy/trunk/scrapy/command/commands/getattr.py new file mode 100644 index 000000000..ac639bf3d --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/getattr.py @@ -0,0 +1,25 @@ +from scrapy.command import ScrapyCommand +from scrapy.fetcher import fetch +from scrapy.spider import spiders +from scrapy.item import ScrapedItem + +def get_item_attr(pagedata, attr="guid"): + spider = spiders.fromurl(pagedata.url) + items = spider.parse(pagedata) + attrs = [getattr(i, attr) for i in items if isinstance(i, ScrapedItem)] + return attrs + +def get_attr(url, attr="guid"): + pagedatas = fetch([url]) + if pagedatas: + return get_item_attr(pagedatas[0], attr) + +class Command(ScrapyCommand): + def syntax(self): + return " " + + def short_desc(self): + return "Print an attribute from the item scraped in the given URL" + + def run(self, args, opts): + print get_attr(args[0], args[1]) diff --git a/scrapy/trunk/scrapy/command/commands/help.py b/scrapy/trunk/scrapy/command/commands/help.py new file mode 100644 index 000000000..8a7358763 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/help.py @@ -0,0 +1,26 @@ +from scrapy.command import ScrapyCommand, cmdline + +class Command(ScrapyCommand): + def syntax(self): + return "" + + def short_desc(self): + return "Provides extended help for the given command" + + def run(self, args, opts): + if not args: + return False + + commands = cmdline.builtin_commands_dict() + commands.update(cmdline.custom_commands_dict()) + + cmdname = args[0] + if cmdname in commands: + cmd = commands[cmdname] + help = getattr(cmd, 'help', None) or getattr(cmd, 'long_desc', None) + print "%s: %s" % (cmdname, cmd.short_desc()) + print "usage: %s %s" % (cmdname, cmd.syntax()) + print + print help() + else: + print "Unknown command: %s" % cmdname diff --git a/scrapy/trunk/scrapy/command/commands/list.py b/scrapy/trunk/scrapy/command/commands/list.py new file mode 100644 index 000000000..70f6f43c4 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/list.py @@ -0,0 +1,16 @@ +from scrapy.command import ScrapyCommand +from scrapy.spider import spiders + +class Command(ScrapyCommand): + def syntax(self): + return "" + + def short_desc(self): + return "List available spiders (both enabled and disabled)" + + def run(self, args, opts): + spiders_dict = spiders.asdict() + for n, p in spiders_dict.items(): + disabled = "disabled" if getattr(p, 'disabled', False) else "enabled" + print "%-30s %-30s %s" % (n, p.__class__.__name__, disabled) + print "Total spiders: %d" % len(spiders_dict) diff --git a/scrapy/trunk/scrapy/command/commands/log.py b/scrapy/trunk/scrapy/command/commands/log.py new file mode 100644 index 000000000..fff7ebbd8 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/log.py @@ -0,0 +1,46 @@ +import re + +from scrapy.command import ScrapyCommand + +log_crawled_re = re.compile(r'^(.*?) \[\w+/(.*?)\].*Crawled <(.*?)> from <(.*?)>$') +log_scraped_re = re.compile('^(.*?) \[\w+/(.*?)\].*Scraped (.*) in <(.*?)>$') +log_opendomain_re = re.compile('^(.*?) \[\w+.*? Started scraping (.*)$') +log_closedomain_re = re.compile('^(.*?) \[\w+.*? Finished scraping (.*)$') +log_debug_re = re.compile(r'\[\w+/(.*?)\] DEBUG') +log_info_re = re.compile(r'\[\w+/(.*?)\] INFO') +log_warning_re = re.compile(r'\[\w+/(.*?)\] WARNING') +log_error_re = re.compile(r'\[\w+/(.*?)\] ERROR') + +class Command(ScrapyCommand): + def syntax(self): + return "[options] " + + def short_desc(self): + return "Several tools to perform scrapy log analysis" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("--sitemap", dest="sitemap", action="store_true", help="show sitemap based on given log file") + + def sitemap(self, logfile): + from scrapy.utils.datatypes import Sitemap + + sm = Sitemap() + for l in open(logfile).readlines(): + m = log_crawled_re.search(l.strip()) + if m: + sm.add_node(m.group(3), m.group(4)) + m = log_scraped_re.search(l.strip()) + if m: + sm.add_item(m.group(4), m.group(3)) + print sm.to_string() + + def run(self, args, opts): + if not args: + print "A log file is required" + return + + if opts.sitemap: + self.sitemap(args[0]) + else: + print "No analysis method specified" diff --git a/scrapy/trunk/scrapy/command/commands/parse.py b/scrapy/trunk/scrapy/command/commands/parse.py new file mode 100644 index 000000000..79860b9af --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/parse.py @@ -0,0 +1,46 @@ +from scrapy.command import ScrapyCommand +from scrapy.fetcher import fetch +from scrapy.http import Request +from scrapy.item import ScrapedItem +from scrapy.spider import spiders +from scrapy.utils import display + +class Command(ScrapyCommand): + def syntax(self): + return "[options] " + + def short_desc(self): + return "Parse the URL and print their results" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("--nolinks", dest="nolinks", action="store_true", help="don't show extracted links") + parser.add_option("--noitems", dest="noitems", action="store_true", help="don't show scraped items") + parser.add_option("--identify", dest="identify", action="store_true", help="try to use identify instead of parse") + parser.add_option("--nocolour", dest="nocolour", action="store_true", help="avoid using pygments to colorize the output") + + def pipeline_process(self, item, opts): + return item + + def run(self, args, opts): + if not args: + print "A URL is required" + return + + responses = fetch([args[0]]) + if responses: + response = responses[0] + spider = spiders.fromurl(response.url) + result = spider.parse(response) if not opts.identify else spider.identify(response) + + items = [self.pipeline_process(i, opts) for i in result if isinstance(i, ScrapedItem)] + links = [i for i in result if isinstance(i, Request)] + + display.nocolour = opts.nocolour + if not opts.noitems: + print "# Scraped Items", "-"*60 + display.pprint(items) + + if not opts.nolinks: + print "# Links", "-"*68 + display.pprint(links) diff --git a/scrapy/trunk/scrapy/command/commands/replay.py b/scrapy/trunk/scrapy/command/commands/replay.py new file mode 100644 index 000000000..a60b180c3 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/replay.py @@ -0,0 +1,174 @@ +import os + +from scrapy.command import ScrapyCommand +from scrapy.replay import Replay +from scrapy.utils import display +from scrapy.conf import settings + +class Command(ScrapyCommand): + def syntax(self): + return "[options] [action]" + + def short_desc(self): + return "Replay a session previously recorded with crawl --record" + + def help(self): + s = "Replay a session previously recorded with crawl --record\n" + s += "\n" + s += "Available actions:\n" + s += " crawl: just replay the crawl (default if action omitted)\n" + s += " diff: replay the crawl and show differences in items scraped/passed\n" + s += " update: replay the crawl and update both scraped and passed items\n" + s += " showitems: show items stored\n" + s += " showpages: show all responses downloaded (not only HTML pages)\n" + return s + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="show verbose output (full items/responses)") + parser.add_option("-t", "--item-type", dest="itype", help="item type (scraped, passed). default: scraped", metavar="TYPE") + parser.add_option("--output", dest="outfile", help="write output to FILE. if omitted uses stdout", metavar="FILE") + parser.add_option("--nocolour", dest="nocolour", action="store_true", help="disable colorized output (for console only)") + parser.add_option("-i", "--ignore", dest="ignores", action="append", help="item attribute to ignore. can be passed multiple times", metavar="ATTR") + parser.add_option("--target", dest="targets", action="append", help="crawl TARGET instead of recorded urls/domains. can be passed multiple times") + + def process_options(self, args, opts): + ScrapyCommand.process_options(self, args, opts) + self.opts = opts + self.action = args[1] if len(args) > 1 else 'crawl' + mode = 'update' if self.action == 'update' else 'play' + usedir = args and os.path.isdir(args[0]) + self.replay = Replay(args[0], mode=mode, usedir=usedir) + if self.action not in ['crawl', 'diff', 'update']: + settings.overrides['LOG_ENABLED'] = False + + def run(self, args, opts): + if not args: + print "A is required" + return + + display.nocolour = opts.nocolour + + if opts.itype == 'passed': + self.before_db = self.replay.passed_old + self.now_db = self.replay.passed_new + else: # default is 'scraped' + opts.itype = 'scraped' + self.before_db = self.replay.scraped_old + self.now_db = self.replay.scraped_new + + actionfunc = getattr(self, 'action_%s' % self.action, None) + if actionfunc: + rep = actionfunc(opts) + if rep: + if opts.outfile: + f = open(opts.outfile, "w") + f.write(rep) + f.close() + else: + print rep, + self.replay.cleanup() + else: + print "Unknown replay action: %s" % self.action + + def action_crawl(self, opts): + self.replay.play(args=opts.targets) + + def action_update(self, opts): + self.action_crawl(opts) + + def action_showitems(self, opts): + s = "" + s += self._format_items(self.before_db.values()) + s += ">>> Total: %d items %s\n" % (len(self.before_db), opts.itype) + return s + + def action_showpages(self, opts): + s = "" + if self.opts.verbose: + for r in self.replay.responses_old.values(): + s += ">>> %s\n" % str(r) + s += display.pformat(r) + else: + s += "\n".join([str(r) for r in self.replay.responses_old.values()]) + "\n" + s += ">>> Total: %d responses downloaded\n" % len(self.replay.responses_old) + return s + + def action_diff(self, opts): + self.action_crawl(opts) + + guids_before = set(self.before_db.keys()) + guids_now = set(map(str, self.now_db.keys())) + + guids_new = guids_now - guids_before + guids_missing = guids_before - guids_now + guids_both = guids_now & guids_before + + chcount, chreport = self._report_differences(self.before_db, self.now_db, guids_both) + + s = "CRAWLING DIFFERENCES REPORT\n\n" + + s += "Total items : %d\n" % (len(guids_both) - chcount) + s += " Items OK : %d\n" % (len(guids_both) - chcount) + s += " New items : %d\n" % len(guids_new) + s += " Missing items : %d\n" % len(guids_missing) + s += " Changed items : %d\n" % chcount + + s += "\n" + s += "- NEW ITEMS (%d) -----------------------------------------\n" % len(guids_new) + s += self._format_items([self.now_db[g] for g in guids_new]) + + s += "\n" + s += "- MISSING ITEMS (%d) -------------------------------------\n" % len(guids_missing) + s += self._format_items([self.before_db[g] for g in guids_missing]) + + s += "\n" + s += "- CHANGED ITEMS (%d) -------------------------------------\n" % chcount + s += chreport + + return s + + def _report_differences(self, old_items, new_items, guids): + items_old = [old_items[g] for g in guids] + items_new = [new_items[g] for g in guids] + + c = 0 + s = "" + for old, new in zip(items_old, items_new): + d = self._item_diff(old, new) + if d: + c += 1 + s += d + return c, s + + def _item_diff(self, old, new): + olddic = old.__dict__ + newdic = new.__dict__ + ignored_attrs = set(self.opts.ignores or []) + allattrs = [a for a in olddic.keys() + newdic.keys() if not a.startswith('_') and a not in ignored_attrs] + + diff = {} + for attr in sorted(allattrs): + oldval = olddic.get(attr, None) + newval = newdic.get(attr, None) + if oldval != newval: + diff[attr] = {} + diff[attr]['old'] = oldval + diff[attr]['new'] = newval + + s = "" + if diff: + s += ">>> Item guid=%s name=%s\n" % (old.guid, old.name) + s += display.pformat(diff) + "\n" + return s + + def _format_items(self, items): + if self.opts.verbose: + s = display.pformat(items) + else: + s = "" + for i in items: + s += "%s\n" % str(i) + s += " <%s>\n" % i.url + return s + diff --git a/scrapy/trunk/scrapy/command/commands/scrape.py b/scrapy/trunk/scrapy/command/commands/scrape.py new file mode 100644 index 000000000..563271f52 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/scrape.py @@ -0,0 +1,81 @@ +import scrapy +from scrapy.command import ScrapyCommand +from scrapy.fetcher import fetch +from scrapy.spider import spiders +from scrapy.xpath import XPathSelector +from scrapy.utils.misc import load_class +from scrapy.extension import extensions +from scrapy.conf import settings + +def load_url(url, response): + vars = {} + itemcls = load_class(settings['DEFAULT_ITEM_CLASS']) + item = itemcls() + vars['item'] = item + vars['url'] = url + vars['response'] = response + vars['xs'] = XPathSelector(response) + vars['spider'] = spiders.fromurl(url) + return vars + +def print_vars(vars): + print '-' * 78 + print "Available local variables:" + for key, val in vars.iteritems(): + if isinstance(val, basestring): + print " %s: %s" % (key, val) + else: + print " %s: %s" % (key, val.__class__) + print '-' * 78 + +class Command(ScrapyCommand): + def syntax(self): + return "" + + def short_desc(self): + return "Interactive scraping console" + + def long_desc(self): + return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html" + + def update_vars(self, vars): + """ You can use this function to update the local variables that will be available in the scrape console """ + pass + + def run(self, args, opts): + if not args: + return False + url = args[0] + + print "Scrapy %s - Interactive scraping console\n" % scrapy.__version__ + + print "Enabling Scrapy extensions...", + extensions.load() + print "done" + print "Downloading URL... ", + responses = fetch([url]) + print "done" + if not responses: + if not opts.loglevel: + print 'Nothing downloaded, run with -o DEBUG to see why it failed' + return + response = responses[0] + + vars = load_url(url, response) + self.update_vars(vars) + print_vars(vars) + + try: # use IPython if available + import IPython + shell = IPython.Shell.IPShell(argv=[], user_ns=vars) + shell.mainloop() + except ImportError: + import code + try: # readline module is only available on unix systems + import readline + except ImportError: + pass + else: + import rlcompleter + readline.parse_and_bind("tab:complete") + code.interact(local=vars) diff --git a/scrapy/trunk/scrapy/command/commands/start.py b/scrapy/trunk/scrapy/command/commands/start.py new file mode 100644 index 000000000..fb8786df2 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/start.py @@ -0,0 +1,16 @@ +from scrapy.command import ScrapyCommand +from scrapy.core.manager import scrapymanager + + +class Command(ScrapyCommand): + def syntax(self): + return "[options]" + + def short_desc(self): + return "Start the Scrapy server" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + + def run(self, args, opts): + scrapymanager.start(*args, **opts.__dict__) diff --git a/scrapy/trunk/scrapy/command/commands/stats.py b/scrapy/trunk/scrapy/command/commands/stats.py new file mode 100644 index 000000000..e062ce6a8 --- /dev/null +++ b/scrapy/trunk/scrapy/command/commands/stats.py @@ -0,0 +1,30 @@ +import pprint +from scrapy.command import ScrapyCommand +from scrapy.conf import settings + +class Command(ScrapyCommand): + def syntax(self): + return " [domain ...]" + + def short_desc(self): + return "Show all stats history stored for the given domain(s)" + + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-p", "--path", dest="path", help="restrict stats to PATH", metavar="PATH") + + def run(self, args, opts): + if not args: + print "A domain is required" + return + if not settings['SCRAPING_DB']: + print "SCRAPING_DB setting is required for this command" + return + + from scrapy.store.db import DomainDataHistory + ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') + + for domain in args: + print "# %s" % domain + pprint.pprint(list(ddh.getall(domain, opts.path))) + diff --git a/scrapy/trunk/scrapy/command/models.py b/scrapy/trunk/scrapy/command/models.py new file mode 100644 index 000000000..57ea715e2 --- /dev/null +++ b/scrapy/trunk/scrapy/command/models.py @@ -0,0 +1,67 @@ +""" +Base class for Scrapy commands +""" +from scrapy.conf import settings + +class ScrapyCommand(object): + def syntax(self): + """ + Command syntax (preferably one-line). Do not include command name. + """ + return "" + + def short_desc(self): + """ + A short description of the command + """ + return "" + + def long_desc(self): + """ + A long description of the command. Return short description when not + available. It cannot contain newlines, since contents will be formatted + by optparser which removes newlines and wraps text. + """ + return self.short_desc() + + def help(self): + """ + An extensive help for the command. It will be shown when using the + "help" command. It can contain newlines, since not post-formatting will + be applied to its contents. + """ + return self.long_desc() + + def add_options(self, parser): + """ + Populate option parse with options available for this command + """ + parser.add_option("-f", "--logfile", dest="logfile", help="logfile to use. if omitted stderr will be used", metavar="FILE") + parser.add_option("-o", "--loglevel", dest="loglevel", default=None, help="log level") + parser.add_option("--spider", dest="spider", default=None, help="default spider (domain) to use if no spider is found") + parser.add_option("--nolog", dest="nolog", action="store_true", help="disable all log messages") + parser.add_option("--profile", dest="profile", default=None, help="write profiling stats in FILE, to analyze later with: python -m pstats FILE", metavar="FILE") + + def process_options(self, args, opts): + if opts.logfile: + settings.overrides['LOG_ENABLED'] = True + settings.overrides['LOGFILE'] = opts.logfile + + if opts.loglevel: + settings.overrides['LOG_ENABLED'] = True + settings.overrides['LOGLEVEL'] = opts.loglevel + + if opts.nolog: + settings.overrides['LOG_ENABLED'] = False + + if opts.spider: + from scrapy.spider import spiders + spiders.default_domain = opts.spider + + def run(self, args, opts): + """ + Entry point for running commands + """ + raise NotImplementedError + + diff --git a/scrapy/trunk/scrapy/conf/__init__.py b/scrapy/trunk/scrapy/conf/__init__.py new file mode 100644 index 000000000..dbe42e26b --- /dev/null +++ b/scrapy/trunk/scrapy/conf/__init__.py @@ -0,0 +1,71 @@ +import os + +SETTINGS_MODULE = os.environ.get('SCRAPYSETTINGS_MODULE', 'scrapy_settings') + +class Settings(object): + """Class to obtain configuration values from settings module + which can be overriden by environment variables prepended by SCRAPY_""" + + # settings in precedence order + overrides = None + settings = None + defaults = None + core = None + + def __init__(self): + self.overrides = {} + self.settings = self._import(SETTINGS_MODULE) + self.defaults = {} + self.core = self._import('scrapy.conf.core_settings') + + def _import(self, modulepath): + try: + return __import__(modulepath, {}, {}, ['']) + except ImportError: + import sys + err = "Error: Can't find %s module in your python path\n" + sys.stderr.write(err % modulepath) + sys.exit(1) + + def __getitem__(self, opt_name): + if opt_name in self.overrides: + return self.overrides[opt_name] + + if 'SCRAPY_' + opt_name in os.environ: + return os.environ['SCRAPY_' + opt_name] + + if hasattr(self.settings, opt_name): + return getattr(self.settings, opt_name) + + if opt_name in self.defaults: + return self.defaults[opt_name] + + if hasattr(self.core, opt_name): + return getattr(self.core, opt_name) + + def get(self, name, default=None): + return self[name] if self[name] is not None else default + + def getbool(self, name, default=False): + """ + True is: 1, '1', True + False is: 0, '0', False, None + """ + return bool(int(self.get(name, default))) + + def getint(self, name, default=0): + return int(self.get(name, default)) + + def getfloat(self, name, default=0.0): + return float(self.get(name, default)) + + def getlist(self, name, default=None): + value = self.get(name) + if value is None: + return [] + elif hasattr(value, '__iter__'): + return value + else: + return str(value).split(',') + +settings = Settings() diff --git a/scrapy/trunk/scrapy/conf/commands/__init__.py b/scrapy/trunk/scrapy/conf/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/conf/commands/crawl.py b/scrapy/trunk/scrapy/conf/commands/crawl.py new file mode 100644 index 000000000..357422ea3 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/commands/crawl.py @@ -0,0 +1 @@ +LOG_STDOUT = True diff --git a/scrapy/trunk/scrapy/conf/commands/help.py b/scrapy/trunk/scrapy/conf/commands/help.py new file mode 100644 index 000000000..bcee80b55 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/commands/help.py @@ -0,0 +1 @@ +LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/list.py b/scrapy/trunk/scrapy/conf/commands/list.py new file mode 100644 index 000000000..bcee80b55 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/commands/list.py @@ -0,0 +1 @@ +LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/log.py b/scrapy/trunk/scrapy/conf/commands/log.py new file mode 100644 index 000000000..bcee80b55 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/commands/log.py @@ -0,0 +1 @@ +LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/scrape.py b/scrapy/trunk/scrapy/conf/commands/scrape.py new file mode 100644 index 000000000..96afeec1a --- /dev/null +++ b/scrapy/trunk/scrapy/conf/commands/scrape.py @@ -0,0 +1,2 @@ +LOG_ENABLED = False + diff --git a/scrapy/trunk/scrapy/conf/commands/stats.py b/scrapy/trunk/scrapy/conf/commands/stats.py new file mode 100644 index 000000000..bcee80b55 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/commands/stats.py @@ -0,0 +1 @@ +LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/commands/test.py b/scrapy/trunk/scrapy/conf/commands/test.py new file mode 100644 index 000000000..bcee80b55 --- /dev/null +++ b/scrapy/trunk/scrapy/conf/commands/test.py @@ -0,0 +1 @@ +LOG_ENABLED = False diff --git a/scrapy/trunk/scrapy/conf/core_settings.py b/scrapy/trunk/scrapy/conf/core_settings.py new file mode 100644 index 000000000..74501c26c --- /dev/null +++ b/scrapy/trunk/scrapy/conf/core_settings.py @@ -0,0 +1,29 @@ +import scrapy +# Scrapy core settings + +BOT_NAME = 'scrapy' +BOT_VERSION = scrapy.__version__ + +ENGINE_DEBUG = False + +# Download configuration options +USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) +DOWNLOAD_TIMEOUT = 180 # 3mins +CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel +REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain + +LOG_ENABLED = True # +LOGLEVEL = 'DEBUG' # default loglevel +LOGFILE = None # None means sys.stderr by default +LOG_STDOUT = False # + +DEFAULT_ITEM_CLASS = 'scrapy.item.ScrapedItem' +SCHEDULER = 'scrapy.core.scheduler.Scheduler' +MEMORYSTORE = 'scrapy.core.scheduler.MemoryStore' +PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer' + +EXTENSIONS = [] + +# contrib.middleware.retry.RetryMiddleware default settings +RETRY_TIMES = 3 +RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] diff --git a/scrapy/trunk/scrapy/contrib/__init__.py b/scrapy/trunk/scrapy/contrib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/closedomain.py b/scrapy/trunk/scrapy/contrib/closedomain.py new file mode 100644 index 000000000..182cbd1ff --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/closedomain.py @@ -0,0 +1,45 @@ +""" +CloseDomain is an extension that forces spiders to be closed after a given +time has expired. + +""" +import datetime +import pprint + +from pydispatch import dispatcher + +from scrapy.core import signals, log +from scrapy.core.engine import scrapyengine +from scrapy.core.mail import MailSender +from scrapy.core.exceptions import NotConfigured +from scrapy.stats import stats +from scrapy.conf import settings + +class CloseDomain(object): + def __init__(self): + self.timeout = settings.getint('CLOSEDOMAIN_TIMEOUT') + if not self.timeout: + raise NotConfigured + + self.tasks = {} + self.mail = MailSender() + self.notify = settings.getlist('CLOSEDOMAIN_NOTIFY') + + dispatcher.connect(self.domain_opened, signal=signals.domain_opened) + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + + def domain_opened(self, domain): + self.tasks[domain] = scrapyengine.addtask(self.close_domain, self.timeout, args=[domain]) + + def close_domain(self, domain): + log.msg("Domain was opened for more than %d seconds, closing it..." % self.timeout, domain=domain) + scrapyengine.close_domain(domain) + if self.notify: + body = "Closed domain %s because it remained opened for more than %s\n\n" % (domain, datetime.timedelta(seconds=self.timeout)) + body += "DOMAIN STATS ------------------------------------------------------\n\n" + body += pprint.pformat(stats.get(domain, None)) + subj = "Closed domain by timeout: %s" % domain + self.mail.send(self.notify, subj, body) + + def domain_closed(self, domain): + scrapyengine.removetask(self.tasks[domain]) diff --git a/scrapy/trunk/scrapy/contrib/cluster/__init__.py b/scrapy/trunk/scrapy/contrib/cluster/__init__.py new file mode 100644 index 000000000..5c544146d --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/cluster/__init__.py @@ -0,0 +1,2 @@ +from scrapy.contrib.cluster.master import ClusterMaster, ClusterMasterWeb +from scrapy.contrib.cluster.worker import ClusterWorker, ClusterWorkerWeb diff --git a/scrapy/trunk/scrapy/contrib/cluster/master/__init__.py b/scrapy/trunk/scrapy/contrib/cluster/master/__init__.py new file mode 100644 index 000000000..fdb4d71b9 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/cluster/master/__init__.py @@ -0,0 +1,2 @@ +from scrapy.contrib.cluster.master.manager import ClusterMaster +from scrapy.contrib.cluster.master.web import ClusterMasterWeb diff --git a/scrapy/trunk/scrapy/contrib/cluster/master/manager.py b/scrapy/trunk/scrapy/contrib/cluster/master/manager.py new file mode 100644 index 000000000..32a5f9239 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/cluster/master/manager.py @@ -0,0 +1,189 @@ +import urlparse +import urllib +import bisect + +from pydispatch import dispatcher +from twisted.web.client import getPage + +from scrapy.core import log, signals +from scrapy.core.engine import scrapyengine +from scrapy.core.exceptions import NotConfigured +from scrapy.utils.serialization import unserialize +from scrapy.conf import settings + +class ClusterNode(object): + def __init__(self, name, url): + self.name = name + self.url = url + self.maxproc = 0 + self.running = {} + self.pending = [] + self.loadavg = (0.0, 0.0, 0.0) + self.status = "down" # down/crawling/idle/error + self.available = False + + self.wsurl = urlparse.urljoin(self.url, "cluster_worker/ws/?format=json") + + def update(self): + d = getPage(self.wsurl) + d.addCallbacks(self._cbUpdate, self._ebUpdate) + + def schedule(self, domains): + args = [("schedule", domain) for domain in domains] + self._wsRequest(args) + + def stop(self, domains): + args = [("stop", domain) for domain in domains] + self._wsRequest(args) + + def remove(self, domains): + args = [("remove", domain) for domain in domains] + self._wsRequest(args) + + def _wsRequest(self, args): + d = getPage("%s?%s" % (self.wsurl, urllib.urlencode(args))) + d.addCallbacks(self._cbUpdate, self._ebUpdate) + + def _cbUpdate(self, jsonstatus): + try: + pmstatus = unserialize(jsonstatus, 'json') + self.maxproc = int(pmstatus.get('maxproc', None)) + self.running = pmstatus.get('running') or {} + self.pending = pmstatus.get('pending') or [] + self.loadavg = tuple(pmstatus.get('loadavg', (0.0, 0.0, 0.0))) + self.status = "crawling" if self.running else "idle" + self.available = True + except ValueError: + self.status = "error" + self.available = False + + def _ebUpdate(self, err): + self.status = "down" + self.available = False + + +class ClusterMaster(object): + + def __init__(self): + if not settings.getbool('CLUSTER_MASTER_ENABLED'): + raise NotConfigured + self.nodes = {} + + dispatcher.connect(self._engine_started, signal=signals.engine_started) + + def load_nodes(self): + """Loads nodes from the CLUSTER_MASTER_NODES setting""" + for name, url in settings.get('CLUSTER_MASTER_NODES', {}).iteritems(): + self.add_node(name, url) + + def update_nodes(self): + for node in self.nodes.itervalues(): + node.update() + + def add_node(self, name, url): + """Add node given its node""" + if name not in self.nodes: + node = ClusterNode(name, url) + self.nodes[name] = node + node.update() + + def remove_node(self, nodename): + raise NotImplemented + + def schedule(self, domains, nodename=None): + if nodename: + node = self.nodes[nodename] + node.schedule(domains) + else: + self._dispatch_domains(domains) + + def stop(self, domains): + to_stop = {} + for domain in domains: + node = self.running.get(domain, None) + if node: + if node.name not in to_stop: + to_stop[node.name] = [] + to_stop[node.name].append(domain) + + for nodename, domains in to_stop.iteritems(): + self.nodes[nodename].stop(domains) + + def remove(self, domains): + to_remove = {} + for domain in domains: + node = self.pending.get(domain, None) + if node: + if node.name not in to_remove: + to_remove[node.name] = [] + to_remove[node.name].append(domain) + + for nodename, domains in to_remove.iteritems(): + self.nodes[nodename].remove(domains) + + def discard(self, domains): + """Stop and remove all running and pending instances of the given + domains""" + self.remove(domains) + self.stop(domains) + + @property + def running(self): + """Return dict of running domains as domain -> node""" + d = {} + for node in self.nodes.itervalues(): + for domain in node.running.iterkeys(): + d[domain] = node + return d + + @property + def pending(self): + """Return dict of pending domains as domain -> node""" + d = {} + for node in self.nodes.itervalues(): + for domain in node.pending: + d[domain] = node + return d + + @property + def available_nodes(self): + return (node for node in self.nodes.itervalues() if node.available) + + def _dispatch_domains(self, domains): + """Schedule the given domains in the availables nodes as good as + possible. The algorithm follows the next rules (in order): + + 1. search for nodes with available capacity(running < maxproc) and (if + any) schedules the domains there + + 2. if there isn't any node with available capacity it schedules the + domain in the node with the smallest number of pending spiders + """ + + to_schedule = {} # domains to schedule per node + pending_node = [] # list of #pending, node + + for node in self.available_nodes: + capacity = node.maxproc - len(node.running) + bisect.insort(pending_node, (len(node.pending), node)) + + to_schedule[node.name] = [] + while domains and capacity > 0: + to_schedule[node.name].append(domains.pop(0)) + capacity -= 1 + if not domains: + break + + for domain in domains: + pending, node = pending_node.pop(0) + to_schedule[node.name].append(domain) + bisect.insort(pending_node, (pending+1, node)) + + for nodename, domains in to_schedule.iteritems(): + if domains: + self.nodes[nodename].schedule(domains) + + def _engine_started(self): + self.load_nodes() + scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL')) + diff --git a/scrapy/trunk/scrapy/contrib/cluster/master/web.py b/scrapy/trunk/scrapy/contrib/cluster/master/web.py new file mode 100644 index 000000000..984d1ba8a --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/cluster/master/web.py @@ -0,0 +1,165 @@ +import datetime + +from pydispatch import dispatcher + +from scrapy.spider import spiders +from scrapy.management.web import banner, webconsole_discover_module +from scrapy.utils.serialization import parse_jsondatetime +from scrapy.contrib.cluster.master import ClusterMaster + +class ClusterMasterWeb(ClusterMaster): + webconsole_id = 'cluster_master' + webconsole_name = 'Cluster master' + + def __init__(self): + ClusterMaster.__init__(self) + + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def webconsole_render(self, wc_request): + changes = "" + if wc_request.path == '/cluster/nodes/': + return self.render_nodes(wc_request) + elif wc_request.path == '/cluster/domains/': + return self.render_domains(wc_request) + elif wc_request.args: + changes = self.webconsole_control(wc_request) + + s = self.render_header() + + s += "

Home

\n" + + s += "\n" + s += "\n" + for node in self.nodes.itervalues(): + #chkbox = "" % domain if node.status in ["up", "idle"] else " " + nodelink = "%s" % (node.name, node.name) + chkbox = " " + loadavg = "%.2f %.2f %.2f" % node.loadavg + s += "\n" % \ + (chkbox, nodelink, node.status, len(node.running), node.maxproc, len(node.pending), loadavg) + s += "
 NameStatusRunningPendingLoad.avg
%s%s%s%d/%d%d%s
\n" + + s += "\n" + s += "\n" + + return str(s) + + def webconsole_control(self, wc_request): + args = wc_request.args + + if "updatenodes" in args: + self.update_nodes() + + if "schedule" in args: + node = args["node"][0] if "node" in args else None + self.schedule(args["schedule"], nodename=node) + + if "stop" in args: + self.stop(args["stop"]) + + if "remove" in args: + self.remove(args["remove"]) + + return "" + + def render_nodes(self, wc_request): + if wc_request.args: + self.webconsole_control(wc_request) + + now = datetime.datetime.utcnow() + + s = self.render_header() + for node in self.nodes.itervalues(): + s += "

%s

\n" % (node.name, node.name) + + s += "

Running domains

\n" + if node.running: + s += "
\n" + s += "\n" + s += "\n" + for domain, proc in node.running.iteritems(): + chkbox = "" % domain if proc['status'] == "running" else " " + start_time = parse_jsondatetime(proc.get('start_time', None)) + elapsed = now - start_time if start_time else None + s += "\n" % \ + (chkbox, proc['pid'], domain, proc['status'], elapsed, proc['logfile']) + s += "
 PIDDomainStatusRunning timeLog file
%s%s%s%s%s%s
\n" + s += "\n" % node.name + s += "

\n" % node.name + s += "
\n" + else: + s += "

No running domains on %s

\n" % node.name + + # pending domains + s += "

Pending domains

\n" + if node.pending: + s += "
\n" + s += "\n" + s += "\n" % node.name + s += "

\n" % node.name + s += "
\n" + else: + s += "

No pending domains on %s

\n" % node.name + + return str(s) + + def render_domains(self, wc_request): + if wc_request.args: + self.webconsole_control(wc_request) + + enabled_domains = set(spiders.asdict(include_disabled=False).keys()) + inactive_domains = enabled_domains - set(self.running.keys() + self.pending.keys()) + + s = self.render_header() + + s += "

Schedule domains

\n" + + s += "Inactive domains (not running or pending)
" + s += "
\n" + s += "\n" + s += "
\n" + s += "Node (only available nodes shown):
\n" + s += "\n" + s += "

\n" + s += "
\n" + + s += "

Domains

\n" + + s += "\n" + s += "\n" + s += self._domains_table(self.running, 'running') + s += self._domains_table(self.pending, 'pending') + s += "
DomainStatusNode
\n" + + return str(s) + + def render_header(self): + s = banner(self) + s += "

Nav: " + s += "Home | " + s += "Domains | " + s += "Nodes (update)" + s += "

" + return s + + def _domains_table(self, dict_, status): + s = "" + for domain, node in dict_.iteritems(): + s += "%s%s%s\n" % (domain, status, node.name) + return s + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py b/scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py new file mode 100644 index 000000000..7b97b63ed --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/cluster/worker/__init__.py @@ -0,0 +1,2 @@ +from scrapy.contrib.cluster.worker.manager import ClusterWorker +from scrapy.contrib.cluster.worker.web import ClusterWorkerWeb diff --git a/scrapy/trunk/scrapy/contrib/cluster/worker/manager.py b/scrapy/trunk/scrapy/contrib/cluster/worker/manager.py new file mode 100644 index 000000000..8914c9e0a --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/cluster/worker/manager.py @@ -0,0 +1,96 @@ +import sys +import os +import time +import datetime + +from twisted.internet import protocol, reactor + +from scrapy.core import log +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings + +class ScrapyProcessProtocol(protocol.ProcessProtocol): + def __init__(self, procman, domain, logfile=None): + self.procman = procman + self.domain = domain + self.logfile = logfile + self.start_time = datetime.datetime.utcnow() + self.status = "starting" + self.pid = -1 + + def __str__(self): + return "" % (self.domain, self.pid, self.status) + + def connectionMade(self): + self.pid = self.transport.pid + log.msg("ClusterWorker: started domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile)) + self.transport.closeStdin() + self.status = "running" + + def processEnded(self, status_object): + log.msg("ClusterWorker: finished domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile)) + del self.procman.running[self.domain] + self.procman.next_pending() + + +class ClusterWorker(object): + + def __init__(self): + if not settings.getbool('CLUSTER_WORKER_ENABLED'): + raise NotConfigured + + self.maxproc = settings.getint('CLUSTER_WORKER_MAXPROC') + self.logdir = settings['CLUSTER_WORKER_LOGDIR'] + self.running = {} + self.pending = [] + + def schedule(self, domain): + """Schedule new domain to be crawled in a separate processes""" + + if len(self.running) < self.maxproc and domain not in self.running: + self._run(domain) + else: + self.pending.append(domain) + + def stop(self, domain): + """Stop running domain. For removing pending (not yet started) domains + use remove() instead""" + + if domain in self.running: + proc = self.running[domain] + log.msg("ClusterWorker: Sending shutdown signal to domain=%s, pid=%d" % (domain, proc.pid)) + proc.transport.signalProcess('INT') + proc.status = "closing" + + def remove(self, domain): + """Remove all scheduled instances of the given domain (if it hasn't + started yet). Otherwise use stop()""" + + while domain in self.pending: + self.pending.remove(domain) + + def next_pending(self): + """Run the next domain in the pending list, which is not already running""" + + if len(self.running) >= self.maxproc: + return + for domain in self.pending: + if domain not in self.running: + self._run(domain) + self.pending.remove(domain) + return + + def _run(self, domain): + """Spawn process to run the given domain. Don't call this method + directly. Instead use schedule().""" + + logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log")) + if not os.path.exists(os.path.dirname(logfile)): + os.makedirs(os.path.dirname(logfile)) + scrapy_proc = ScrapyProcessProtocol(self, domain, logfile) + env = {'SCRAPY_LOGFILE': logfile} + args = [sys.executable, sys.argv[0], 'crawl', domain] + proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=env) + self.running[domain] = scrapy_proc + + diff --git a/scrapy/trunk/scrapy/contrib/cluster/worker/web.py b/scrapy/trunk/scrapy/contrib/cluster/worker/web.py new file mode 100644 index 000000000..889d90a3f --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/cluster/worker/web.py @@ -0,0 +1,127 @@ +import os +import datetime + +from pydispatch import dispatcher + +from scrapy.spider import spiders +from scrapy.management.web import banner, webconsole_discover_module +from scrapy.utils.serialization import serialize +from scrapy.contrib.cluster.worker import ClusterWorker + +class ClusterWorkerWeb(ClusterWorker): + webconsole_id = 'cluster_worker' + webconsole_name = 'Cluster worker' + + def __init__(self): + ClusterWorker.__init__(self) + + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def webconsole_render(self, wc_request): + changes = "" + if wc_request.path == '/cluster_worker/ws/': + return self.webconsole_control(wc_request, ws=True) + elif wc_request.args: + changes = self.webconsole_control(wc_request) + + now = datetime.datetime.utcnow() + + s = banner(self) + + # running processes + s += "

Running processes

\n" + if self.running: + s += "
\n" + s += "\n" + s += "\n" + for domain, proc in self.running.iteritems(): + chkbox = "" % domain if proc.status == "running" else " " + elapsed = now - proc.start_time + s += "\n" % \ + (chkbox, proc.pid, domain, proc.status, proc.logfile, elapsed) + s += "
 PIDDomainStatusLog fileRunning time
%s%s%s%s%s%s
\n" + s += "

\n" + s += "
\n" + else: + s += "

No running processes

\n" + + # pending domains + s += "

Pending domains

\n" + if self.pending: + s += "
\n" + s += "\n" + s += "

\n" + s += "
\n" + else: + s += "

No pending domains

\n" + + # schedule domains + enabled_domains = spiders.asdict(include_disabled=False).keys() + s += "

Schedule domains

\n" + s += "
\n" + s += "\n" + s += "

\n" + s += "
\n" + + s += changes + + s += "\n" + s += "\n" + + return s + + def webconsole_control(self, wc_request, ws=False): + args = wc_request.args + + if "schedule" in args: + for domain in args["schedule"]: + self.schedule(domain) + if ws: + return self.ws_status(wc_request) + else: + return "

Scheduled domains:

  • %s
" % "
  • ".join(args["schedule"]) + "

    \n" + + if "stop" in args: + for domain in args["stop"]: + self.stop(domain) + if ws: + return self.ws_status(wc_request) + else: + return "

    Stopped running processes:

    • %s
    " % "
  • ".join(args["stop"]) + "

    \n" + + if "remove" in args: + for domain in args["remove"]: + self.remove(domain) + if ws: + return self.ws_status(wc_request) + else: + return "

    Removed pending domains:

    • %s
    " % "
  • ".join(args["remove"]) + "

    \n" + + if ws: + return self.ws_status(wc_request) + else: + return "" + + def ws_status(self, wc_request): + format = wc_request.args['format'][0] if 'format' in wc_request.args else 'json' + wc_request.setHeader('content-type', 'text/plain') + exported_proc_attrs = ['pid', 'status', 'start_time', 'logfile'] + d = {'maxproc': self.maxproc, 'running': {}, 'pending': []} + for domain, proc in self.running.iteritems(): + d2 = {} + for a in exported_proc_attrs: + d2[a] = getattr(proc, a) + d['running'][domain] = d2 + d['pending'] = self.pending + d['loadavg'] = os.getloadavg() + content = serialize(d, format) + return content + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/debug.py b/scrapy/trunk/scrapy/contrib/debug.py new file mode 100644 index 000000000..6638960a8 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/debug.py @@ -0,0 +1,17 @@ +""" +Extensions for debugging Scrapy +""" +import signal +import traceback + +class StackTraceDebug(object): + def __init__(self): + try: + signal.signal(signal.SIGUSR1, self.dump_stacktrace) + except AttributeError: + # win32 platforms don't support SIGUSR signals + pass + + def dump_stacktrace(self, signum, frame): + print "Got signal. Dumping stack trace..." + traceback.print_stack(frame) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py new file mode 100644 index 000000000..773d3d99d --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py @@ -0,0 +1,173 @@ +from __future__ import with_statement + +import os +import sha +import datetime +import urlparse +from pydispatch import dispatcher +from twisted.internet import defer + +from scrapy.core import signals, log +from scrapy.core.engine import scrapyengine +from scrapy.http import Response, Headers +from scrapy.http.headers import headers_dict_to_raw +from scrapy.core.exceptions import UsageError, NotConfigured, HttpException, IgnoreRequest +from scrapy.conf import settings + + +class CacheMiddleware(object): + def __init__(self): + if not settings['CACHE2_DIR']: + raise NotConfigured + self.cache = Cache(settings['CACHE2_DIR'], sectorize=settings.getbool('CACHE2_SECTORIZE')) + self.ignore_missing = settings.getbool('CACHE2_IGNORE_MISSING') + dispatcher.connect(self.open_domain, signal=signals.domain_open) + + def open_domain(self, domain): + self.cache.open_domain(domain) + + def process_request(self, request, spider): + if not is_cacheable(request): + return + + key = request.fingerprint() + domain = spider.domain_name + response = self.cache.retrieve_response(domain, key) + if response: + response.cached = True + if not 200 <= int(response.status) < 300: + raise HttpException(response.status, None, response) + return response + elif self.ignore_missing: + raise IgnoreRequest("Ignored request not in cache: %s" % request) + + def process_response(self, request, response, spider): + if not is_cacheable(request): + return response + + if isinstance(response, Response) and not response.cached: + key = request.fingerprint() + domain = spider.domain_name + self.cache.store(domain, key, request, response) + + return response + + def process_exception(self, request, exception, spider): + if not is_cacheable(request): + return + + if isinstance(exception, HttpException) and isinstance(exception.response, Response): + key = request.fingerprint() + domain = spider.domain_name + self.cache.store(domain,key, request, exception.response) + +def is_cacheable(request): + scheme, _, _, _, _ = urlparse.urlsplit(request.url) + return scheme in ['http', 'https'] + + +class Cache(object): + DOMAIN_SECTORDIR = 'data' + DOMAIN_LINKDIR = 'domains' + + def __init__(self, cachedir, sectorize=False): + self.cachedir = cachedir + self.sectorize = sectorize + + self.baselinkpath = os.path.join(self.cachedir, self.DOMAIN_LINKDIR) + if not os.path.exists(self.baselinkpath): + os.makedirs(self.baselinkpath) + + self.basesectorpath = os.path.join(self.cachedir, self.DOMAIN_SECTORDIR) + if not os.path.exists(self.basesectorpath): + os.makedirs(self.basesectorpath) + + def domainsectorpath(self, domain): + sector = sha.sha(domain).hexdigest()[0] + return os.path.join(self.basesectorpath, sector, domain) + + def domainlinkpath(self, domain): + return os.path.join(self.baselinkpath, domain) + + def requestpath(self, domain, key): + linkpath = self.domainlinkpath(domain) + return os.path.join(linkpath, key[0:2], key) + + def open_domain(self, domain): + if domain: + linkpath = self.domainlinkpath(domain) + if self.sectorize: + sectorpath = self.domainsectorpath(domain) + if not os.path.exists(sectorpath): + os.makedirs(sectorpath) + if not os.path.exists(linkpath): + try: + os.symlink(sectorpath, linkpath) + except: + os.makedirs(linkpath) # windows filesystem + else: + if not os.path.exists(linkpath): + os.makedirs(linkpath) + + def is_cached(self, domain, key): + requestpath = self.requestpath(domain, key) + return os.path.exists(requestpath) + + def retrieve_response(self, domain, key): + """ + Return response dictionary if request has correspondent cache record; + return None if not. + """ + requestpath = self.requestpath(domain, key) + if not os.path.exists(requestpath): + return None # not cached + + metadata = responsebody = responseheaders = None + with open(os.path.join(requestpath, 'meta_data')) as f: + metadata = f.read() + with open(os.path.join(requestpath, 'response_body')) as f: + responsebody = f.read() + with open(os.path.join(requestpath, 'response_headers')) as f: + responseheaders = f.read() + + metadata = eval(metadata) + url = metadata['url'] + original_url = metadata.get('original_url', url) + headers = Headers(responseheaders) + parent = metadata.get('parent') + status = metadata['status'] + + response = Response(domain=domain, url=url, original_url=original_url, headers=headers, status=status, body=responsebody, parent=parent) + response.cached = True + return response + + def store(self, domain, key, request, response): + requestpath = self.requestpath(domain, key) + if not os.path.exists(requestpath): + os.makedirs(requestpath) + + metadata = { + 'url':request.url, + 'method': request.method, + 'parent': response.parent, + 'status': response.status, + 'domain': response.domain, + 'original_url': response.original_url, + 'timestamp': datetime.datetime.now(), + } + + # metadata + with open(os.path.join(requestpath, 'meta_data'), 'w') as f: + f.write(repr(metadata)) + # response + with open(os.path.join(requestpath, 'response_headers'), 'w') as f: + f.write(headers_dict_to_raw(response.headers)) + with open(os.path.join(requestpath, 'response_body'), 'w') as f: + f.write(response.body.get_content()) + # request + with open(os.path.join(requestpath, 'request_headers'), 'w') as f: + f.write(headers_dict_to_raw(request.headers)) + if request.body: + with open(os.path.join(requestpath, 'request_body'), 'w') as f: + f.write(request.body) + diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py new file mode 100644 index 000000000..063ea72b7 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/common.py @@ -0,0 +1,10 @@ +class CommonMiddleware(object): + """This middleware provides common/basic functionality, and should always + be enabled""" + + def process_request(self, request, spider): + request.headers.setdefault('Accept-Language', 'en') + request.headers.setdefault('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8') + if request.method == 'POST': + request.headers.setdefault('Content-Type', 'application/x-www-form-urlencoded') + diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/compression.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/compression.py new file mode 100644 index 000000000..9b23f33f1 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/compression.py @@ -0,0 +1,42 @@ +import zlib +from gzip import GzipFile +from cStringIO import StringIO + +from scrapy.http import Response, ResponseBody + + +class CompressionMiddleware(object): + """This middleware allows compressed (gzip, deflate) traffic to be + sent/received from web sites""" + + def process_request(self, request, spider): + request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + + def process_response(self, request, response, spider): + if isinstance(response, Response): + content_encoding = response.headers.get('Content-Encoding') + if content_encoding: + encoding = content_encoding[0].lower() + raw_body = response.body.get_content() + declared_encoding = response.body.declared_encoding + decoded_body = self._decode(raw_body, encoding) + response.body = ResponseBody(decoded_body, declared_encoding) + response.headers['Content-Encoding'] = content_encoding[1:] + return response + + def _decode(self, body, encoding): + if encoding == 'gzip': + body = GzipFile(fileobj=StringIO(body)).read() + + if encoding == 'deflate': + try: + body = zlib.decompress(body) + except zlib.error: + # ugly hack to work with raw deflate content that may + # be sent by microsof servers. For more information, see: + # http://carsten.codimi.de/gzip.yaws/ + # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx + # http://www.gzip.org/zlib/zlib_faq.html#faq38 + body = zlib.decompress(body, -15) + return body + diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py new file mode 100644 index 000000000..9816dc521 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cookies.py @@ -0,0 +1,27 @@ +from pydispatch import dispatcher + +from scrapy.core import signals +from scrapy.core.engine import scrapyengine +from scrapy.utils.misc import dict_updatedefault + +class CookiesMiddleware(object): + """This middleware enables working with sites that need cookies""" + + def __init__(self): + self.cookies = {} + dispatcher.connect(self.domain_open, signals.domain_open) + dispatcher.connect(self.domain_closed, signals.domain_closed) + + def process_request(self, request, spider): + dict_updatedefault(request.cookies, self.cookies[spider.domain_name]) + + def process_response(self, request, response, spider): + cookies = self.cookies[spider.domain_name] + cookies.update(request.cookies) + return response + + def domain_open(self, domain): + self.cookies[domain] = {} + + def domain_closed(self, domain): + del self.cookies[domain] diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py new file mode 100644 index 000000000..392e8f9c0 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/debug.py @@ -0,0 +1,20 @@ +from scrapy.core import log +from scrapy.conf import settings + +class CrawlDebug(object): + def __init__(self): + self.enabled = settings.getbool('CRAWL_DEBUG') + + def process_request(self, request, spider): + if self.enabled: + log.msg("Crawling %s" % repr(request), domain=spider.domain_name, level=log.DEBUG) + + def process_exception(self, request, exception, spider): + if self.enabled: + log.msg("Crawl exception %s in %s" % (exception, repr(request)), domain=spider.domain_name, level=log.DEBUG) + + def process_response(self, request, response, spider): + if self.enabled: + log.msg("Fetched %s from %s" % (response.info(), repr(request)), domain=spider.domain_name, level=log.DEBUG) + return response + diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py new file mode 100644 index 000000000..d9d0931ad --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/errorpages.py @@ -0,0 +1,13 @@ +from scrapy.core.exceptions import HttpException + +class ErrorPagesMiddleware(object): + """This middleware allows the spiders to receive error (non 200) responses, + the same way the receive normal responses""" + + def process_exception(self, request, exception, spider): + if isinstance(exception, HttpException): + statuses = getattr(spider, 'handle_httpstatus_list', None) + httpstatus = exception.response.status + if statuses and httpstatus in statuses: + return exception.response + diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py new file mode 100644 index 000000000..fcc9ebb8c --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py @@ -0,0 +1,8 @@ +class HttpAuthMiddleware(object): + """This middleware allows spiders to use HTTP auth in a cleaner way + (http_user and http_pass spider class attributes)""" + + def process_request(self, request, spider): + if getattr(spider, 'http_user', None) or getattr(spider, 'http_pass', None): + request.httpauth(spider.http_user, spider.http_pass) + diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py new file mode 100644 index 000000000..85052c020 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py @@ -0,0 +1,55 @@ +import re + +from scrapy.core import log +from scrapy.http import Request, Response +from scrapy.core.exceptions import HttpException +from scrapy.utils.url import urljoin_rfc as urljoin + +META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);url=([^"\']+)', re.IGNORECASE) +# some sites use meta-refresh for redirecting to a session expired page, so we +# restrict automatic redirection to a maximum delay (in number of seconds) +META_REFRESH_MAXSEC = 100 + +class RedirectMiddleware(object): + def process_exception(self, request, exception, spider): + if isinstance(exception, HttpException): + status = exception.status + response = exception.response + + if status in ['302', '303']: + redirected_url = urljoin(request.url, response.headers['location'][0]) + if not getattr(spider, "no_redirect", False): + redirected = request.copy() + redirected.url = redirected_url + redirected.method = 'GET' + redirected.body = None + # This is needed to avoid redirection loops with requests that contain dont_filter = True + # Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks + redirected.dont_filter = False + log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name) + return redirected + log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name) + return response + + if status in ['301', '307']: + redirected_url = urljoin(request.url, response.headers['location'][0]) + if not getattr(spider, "no_redirect", False): + redirected = request.copy() + redirected.url = redirected_url + # This is needed to avoid redirection loops with requests that contain dont_filter = True + # Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks + redirected.dont_filter = False + log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name) + return redirected + log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name) + return response + + def process_response(self, request, response, spider): + if isinstance(response, Response): + m = META_REFRESH_RE.search(response.body.to_string()[0:4096]) + if m and int(m.group(1)) < META_REFRESH_MAXSEC: + redirected = request.copy() + redirected.url = urljoin(request.url, m.group(2)) + log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name) + return redirected + return response diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py new file mode 100644 index 000000000..ad6e05f7c --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py @@ -0,0 +1,63 @@ +""" +An extension to retry failed requests that are potentially caused by temporary +problems such as a connection timeout or HTTP 500 error. + +You can change the behaviour of this moddileware by modifing the scraping settings: +RETRY_TIMES - how many times to retry a failed page +RETRY_HTTP_CODES - which HTTP response codes to retry + +Failed pages are collected on the scraping process and rescheduled at the end, +once the spider has finished crawling all regular (non failed) pages. Once +there is no more failed pages to retry this middleware sends a signal +(retry_complete), so other extensions could connect to that signal. + +Default values are located in scrapy.conf.core_settings + +About HTTP errors to consider: + +- You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the HTTP + protocol. It's included by default because it's a common code used to + indicate server overload, which would be something we want to retry +- 200 is included by default (and shoudln't be removed) to check for partial + downloads errors, which means the TCP connection has broken in the middle of + a HTTP download +""" + +from twisted.internet.error import TimeoutError as ServerTimeoutError, DNSLookupError, \ + ConnectionRefusedError, ConnectionDone, ConnectError +from twisted.internet.defer import TimeoutError as UserTimeoutError + +from scrapy.core import log +from scrapy.core.exceptions import HttpException +from scrapy.conf import settings + +class RetryMiddleware(object): + + EXCEPTIONS_TO_RETRY = (ServerTimeoutError, UserTimeoutError, DNSLookupError, + ConnectionRefusedError, ConnectionDone, ConnectError) + + def __init__(self): + self.failed_count = {} + self.max_retries = settings.getint('RETRY_TIMES') + + def process_exception(self, request, exception, spider): + retry = False + + if isinstance(exception, self.EXCEPTIONS_TO_RETRY): + retry = True + elif isinstance(exception, HttpException): + if exception.status in settings.getlist('RETRY_HTTP_CODES'): + retry = True + + if retry: + fp = request.fingerprint() + count = self.failed_count[fp] = self.failed_count.get(fp, 0) + 1 + + if self.failed_count[fp] < self.max_retries: + log.msg("Retrying %s (failed %d times): %s" % (request, count, exception), level=log.DEBUG, domain=spider.domain_name) + retryreq = request.copy() + retryreq.dont_filter = True + return retryreq + else: + log.msg("Discarding %s (failed %d times): %s" % (request, count, exception), domain=spider.domain_name, level=log.DEBUG) + diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/robots.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/robots.py new file mode 100644 index 000000000..2c9a0371e --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/robots.py @@ -0,0 +1,59 @@ +""" +jhis is an expertimental middleware to respect robots.txt policies. The biggest +problem it has is that it uses urllib directly (in RobotFileParser.read() +method) and that conflicts with twisted networking, so it should be ported to +use twisted networking API, but that is not as trivial as it may seem. + +This code is left here for future reference, when we resume the work on this +subject. + +""" + +import re +import urlparse +import robotparser + +from pydispatch import dispatcher + +from scrapy.core import log, signals +from scrapy.core.exceptions import IgnoreRequest +from scrapy.conf import settings + +BASEURL_RE = re.compile("http://.*?/") + +class RobotsMiddleware(object): + + def __init__(self): + self._parsers = {} + self._spiderdomains = {} + self._pending = {} + dispatcher.connect(self.domain_open, signals.domain_open) + dispatcher.connect(self.domain_closed, signals.domain_closed) + + def process_request(self, request, spider): + agent = getattr(spider, 'user_agent', None) or settings['USER_AGENT'] + rp = self.robot_parser(request.url, spider.domain_name) + if rp and not rp.can_fetch(agent, request.url): + raise IgnoreRequest("URL forbidden by robots.txt: %s" % request.url) + + def robot_parser(self, url, spiderdomain): + urldomain = urlparse.urlparse(url).hostname + if urldomain in self._parsers: + rp = self._parsers[urldomain] + else: + rp = robotparser.RobotFileParser() + m = BASEURL_RE.search(url) + if m: + rp.set_url("%srobots.txt" % m.group()) + rp.read() + self._parsers[urldomain] = rp + self._spiderdomains[spiderdomain].add(urldomain) + return rp + + def domain_open(self, domain): + self._spiderdomains[domain] = set() + + def domain_closed(self, domain): + for urldomain in self._spiderdomains[domain]: + del self._parsers[urldomain] + del self._spiderdomains[domain] diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py new file mode 100644 index 000000000..6bda3e85f --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/useragent.py @@ -0,0 +1,7 @@ +class UserAgentMiddleware(object): + """This middleware allows spiders to override the user_agent""" + + def process_request(self, request, spider): + if getattr(spider, 'user_agent', None): + request.headers.setdefault('User-Agent', spider.user_agent) + diff --git a/scrapy/trunk/scrapy/contrib/groupsettings.py b/scrapy/trunk/scrapy/contrib/groupsettings.py new file mode 100644 index 000000000..4bad4100b --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/groupsettings.py @@ -0,0 +1,26 @@ +""" +Extensions to override scrapy settings with per-group settings according to the +group the spider belongs to. It only overrides the settings when running the +crawl command with *only one domain as argument*. +""" + +from scrapy.conf import settings +from scrapy.core.exceptions import NotConfigured +from scrapy.command.cmdline import command_executed + +class GroupSettings(object): + + def __init__(self): + if not settings.getbool("GROUPSETTINGS_ENABLED"): + raise NotConfigured + + if command_executed and command_executed['name'] == 'crawl': + mod = __import__(settings['GROUPSETTINGS_MODULE'], {}, {}, ['']) + args = command_executed['args'] + if len(args) == 1 and not args[0].startswith('http://'): + domain = args[0] + settings.overrides.update(mod.default_settings) + for group, domains in mod.group_spiders.iteritems(): + if domain in domains: + settings.overrides.update(mod.group_settings.get(group, {})) + diff --git a/scrapy/trunk/scrapy/contrib/history/__init__.py b/scrapy/trunk/scrapy/contrib/history/__init__.py new file mode 100644 index 000000000..81eb201d6 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/history/__init__.py @@ -0,0 +1,3 @@ +from scrapy.contrib.history.history import ItemHistory +from scrapy.contrib.history.scheduler import RulesScheduler +from scrapy.contrib.history.store import SQLHistoryStore diff --git a/scrapy/trunk/scrapy/contrib/history/history.py b/scrapy/trunk/scrapy/contrib/history/history.py new file mode 100644 index 000000000..6767ef260 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/history/history.py @@ -0,0 +1,162 @@ +""" +History management +""" +import re +from datetime import datetime + +import MySQLdb + +class History(object): + """Base class for tracking different kinds of histories""" + + def __init__(self, db_uri): + self.db_uri = db_uri + self._mysql_conn = None + + def connect(self): + """ + Connect to PDB and open mysql connect to PRODUCT_DB + """ + m = re.search(r"mysql:\/\/(?P[^:]+)(:(?P[^@]+))?@(?P[^/]+)/(?P.*)$", self.db_uri) + if m: + d = m.groupdict() + if d['passwd'] is None: + del(d['passwd']) + + d['charset'] = "utf8" + self._mysql_conn = MySQLdb.connect(**d) + + def get_mysql_conn(self): + if self._mysql_conn is None: + self.connect() + return self._mysql_conn + mysql_conn = property(get_mysql_conn) + + +class ItemHistory(History): + """ + Class instance registers item guids, and versions with check_item method. + It also gives access to stored ItemTicket and ItemVersion objects. + """ + NEW = 0 + UPDATE = 1 + DUPLICATE = 2 + + def check_item(self, domain, item): + """ + Store item's guid and version to the database along + with date and time of last occurence. + Return: + * ItemHistory.NEW - if guid hasn't been met + * ItemHistory.UPDATE - if history already contains guid, but versions doesn't match + * ItemHistory.DUPLICATE - both guid and version are not new + """ + version = item.version + + c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) + + def add_version(version): + insert = "INSERT INTO version (guid, version, seen) VALUES (%s,%s,%s)" + c.execute(insert, (item.guid, version, datetime.now())) + + select = "SELECT * FROM ticket WHERE guid=%s" + c.execute(select, item.guid) + r = c.fetchone() + if r: + select = "SELECT * FROM version WHERE version=%s" + if c.execute(select, version): + update = "UPDATE version SET seen=%s WHERE version=%s" + c.execute(update, (datetime.now(), version)) + self.mysql_conn.commit() + return ItemHistory.DUPLICATE + else: + add_version(version) + self.mysql_conn.commit() + return ItemHistory.UPDATE + else: + insert = "INSERT INTO ticket (guid, domain, url, url_hash) VALUES (%s,%s,%s,%s)" + c.execute(insert, (item.guid, domain, item.url, hash(item.url))) + add_version(version) + self.mysql_conn.commit() + return ItemHistory.NEW + + def get_ticket(self, guid): + """ + Return ItemTicket object for guid. + ItemVersion objects can be accessed via 'versions' list. + """ + c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) + select = "SELECT * FROM ticket WHERE guid=%s" + c.execute(select, guid) + ticket = c.fetchone() + if not ticket: + raise Exception("Item ticket with guid = '%s' not found" % guid) + ticket['versions'] = [] + select = "SELECT * FROM version WHERE guid=%s" + c.execute(select, guid) + for version in c.fetchall(): + ticket['versions'].append(version) + return ticket + + def delete_ticket(self, guid): + """Delete item ticket and associated versions from DB""" + c = self.mysql_conn.cursor() + delete = "DELETE FROM ticket WHERE guid=%s" + c.execute(delete, guid) + self.mysql_conn.commit() + + +class URLHistory(History): + """ + Access URL status and history for the scraping engine + + This is degsigned to have an instance per domain where typically + a call will be made to get_url_status, followed by either + update_checked or record_version. + """ + + def get_url_status(self, urlkey): + """ + Get the url status (url, last_version, last_checked), + or None if the url data has not been seen before + """ + c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) + select = "SELECT * FROM url_status WHERE url_hash=%s" + c.execute(select, urlkey) + r = c.fetchone() + return (r['url'], r['last_version'], r['last_checked']) if r else None + + def record_version(self, urlkey, url, parent_key, version, postdata_hash=None): + """ + Record a version of a page and update the last checked time. + + If the same version (or None) is passed, the last checked time is still updated. + """ + now = datetime.now() + c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) + select = "SELECT * FROM url_status WHERE url_hash=%s" + c.execute(select, urlkey) + r = c.fetchone() + + if not r: + insert = "INSERT INTO url_status (url_hash, url, parent_hash, last_version, last_checked) VALUES (%s,%s,%s,%s,%s)" + c.execute(insert, (urlkey, url, parent_key, version, now)) + else: + update = "UPDATE url_status SET last_version=%s, last_checked=%s WHERE url_hash=%s" + c.execute(update, (version, now, urlkey)) + self.mysql_conn.commit() + + last_version = r['last_version'] if r else None + if version and version != last_version: + if not c.execute("SELECT url_hash FROM url_history WHERE version=%s", version): + insert = "INSERT INTO url_history (url_hash, version, postdata_hash, created) VALUES (%s,%s,%s,%s)" + c.execute(insert, (urlkey, version, postdata_hash, now)) + self.mysql_conn.commit() + + def get_version_info(self, version): + """Simple accessor method""" + c = self.mysql_conn.cursor(MySQLdb.cursors.DictCursor) + select = "SELECT * FROM url_history WHERE version=%s" + c.execute(select, version) + r = c.fetchone() + return (r['url_hash'], r['created']) if r else None diff --git a/scrapy/trunk/scrapy/contrib/history/middleware.py b/scrapy/trunk/scrapy/contrib/history/middleware.py new file mode 100644 index 000000000..45e3f3f51 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/history/middleware.py @@ -0,0 +1,87 @@ +import sha +from datetime import datetime + +from pydispatch import dispatcher + +from scrapy.utils.misc import load_class +from scrapy.core import log, signals +from scrapy.core.exceptions import NotConfigured, IgnoreRequest +from scrapy.conf import settings + +class HistoryMiddleware(object): + # How often we should re-check links we know about + MIN_CHECK_DAYS = 4 + # How often we should process pages that have not changed (need to include depth) + MIN_PROCESS_UNCHANGED_DAYS = 12 + + def __init__(self): + historycls = load_class(settings['MEMORYSTORE']) + if not historycls: + raise NotConfigured + self.historydata = historycls() + dispatcher.connect(self.open_domain, signal=signals.domain_open) + dispatcher.connect(self.close_domain, signal=signals.domain_closed) + + def process_request(self, request, spider): + key = urlkey(request.url) + status = self.historydata.status(domain, key) + if status: + _url, version, last_checked = status + d = datetime.now() - last_checked + if d.days < self.MIN_CHECK_DAYS: + raise IgnoreRequest("Not scraping %s (scraped %s ago)" % (request.url, d)) + request.context['history_response_version'] = version + + def process_response(self, request, response, spider): + version = request.context.get('history_response_version') + if version == response.version(): + del request.content['history_response_version'] + hist = self.historydata.version_info(domain, version) + if hist: + versionkey, created = hist + # if versionkey != urlkey(url) this means + # the same content is available on a different url + delta = datetime.now() - created + if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS: + message = "skipping %s: unchanged for %s" % (response.url, delta) + raise IgnoreRequest(message) + self.record_visit(domain, request, response) + return response + + def process_exception(self, request, exception, spider): + self.record_visit(spider.domain_name, request, None) + + def open_domain(self, domain): + self.historydata.open(domain) + + def close_domain(self, domain): + self.historydata.close_site(domain) + + def record_visit(self, domain, request, response): + """record the fact that the url has been visited""" + url = request.url + post_version = hash(request.body) + key = urlkey(url) + if response: + redirect_url = response.url + parentkey = urlkey(response.parent) if response.parent else None + version = response.version() + else: + redirect_url, parentkey, version = url, None, None + self.historydata.store(domain, key, url, parentkey, version, post_version) + + +def urlkey(url): + """Generate a 'key' for a given url + + >>> urlkey("http://www.example.com/") + '89e6a0649e06d83370cdf2cbfb05f363934a8d0c' + >>> urlkey("http://www.example.com/") == urlkey("http://www.example.com/?") + True + """ + from scrapy.utils.c14n import canonicalize + return hash(canonicalize(url)) + + +def hash(value): + return sha.new(value).hexdigest() if value else None diff --git a/scrapy/trunk/scrapy/contrib/history/scheduler.py b/scrapy/trunk/scrapy/contrib/history/scheduler.py new file mode 100644 index 000000000..eaf562d96 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/history/scheduler.py @@ -0,0 +1,94 @@ +from datetime import datetime + +from twisted.internet import defer + +from scrapy.core import log +from scrapy.core.scheduler import Scheduler +from scrapy.core.exceptions import IgnoreRequest + +class RulesScheduler(Scheduler): + """Scheduler that uses rules to determine if we should follow links + + TODO: + * take into account where in chain of links we are (less depth should + be crawled more often) + * Be more strict about scraping product pages that rarely lead to new + versions of products. The same applies to pages with links. Particularly + useful for filtering out when there are many urls for the same product. + (but be careful to also filter out pages that almost always lead to new + output). + """ + + # if these parameters change, then update bin/unavailable.py + + # How often we should re-check links we know about + MIN_CHECK_DAYS = 4 + + # How often we should process pages that have not changed (need to include depth) + MIN_PROCESS_UNCHANGED_DAYS = 12 + + def enqueue_request(self, domain, request, priority=1): + """Add a page to be scraped for a domain that is currently being scraped. + + The url will only be added if we have not checked it already within + a specified time period. + """ + requestid = request.fingerprint() + added = self.groupfilter.add(domain, requestid) + + if request.dont_filter or added: + key = urlkey(request.url) # we can not use fingerprint unless lost crawled history + status = self.historydata.status(domain, key) + now = datetime.now() + version = None + if status: + _url, version, last_checked = status + d = now - last_checked + if d.days < self.MIN_CHECK_DAYS: + log.msg("Not scraping %s (scraped %s ago)" % (request.url, d), level=log.DEBUG) + return + # put the version in the pending pages to avoid querying DB again + record = (request, version, now) + self.pending_requests[domain].put(record, priority) + + def next_request(self, domain): + """Get the next page from the superclass. This will add a callback + to prevent processing the page unless its content has been + changed. + + In the event that it a page is not processed, the record_visit method + is called to update the last_checked time. + """ + pending_list = self.pending_requests.get(domain) + if not pending_list : + return None + request, version, timestamp = pending_list.get_nowait()[1] + post_version = hash(request.body) + + def callback(pagedata): + """process other callback if we pass the checks""" + if version == pagedata.version(): + hist = self.historydata.version_info(domain, version) + if hist: + versionkey, created = hist + # if versionkey != urlkey(url) this means + # the same content is available on a different url + delta = timestamp - created + if delta.days < self.MIN_PROCESS_UNCHANGED_DAYS: + message = "skipping %s: unchanged for %s" % (pagedata.url, delta) + raise IgnoreRequest(message) + self.record_visit(domain, request.url, pagedata.url, + pagedata.parent, pagedata.version(), + post_version) + return pagedata + + def errback(error) : + self.record_visit(domain, request.url, request.url, None, None, + post_version) + return error + + d = defer.Deferred() + d.addCallbacks(callback, errback) + request.prepend_callback(d) + + return request diff --git a/scrapy/trunk/scrapy/contrib/history/store.py b/scrapy/trunk/scrapy/contrib/history/store.py new file mode 100644 index 000000000..839425fbf --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/history/store.py @@ -0,0 +1,47 @@ +""" +SQLHistoryStore + +Persistent history storage using relational database +""" +from scrapy.contrib.history.history import URLHistory +from scrapy.core import log +from scrapy.conf import settings + +class SQLHistoryStore(object) : + """Implementation of a data store that stores information in a relation + database + + This maintains a URLHistory object per site. That means each domain + has it's own session and is isolated from the others. + """ + + def __init__(self): + self._store = {} + self._dbinfo = settings['SCRAPING_DB'] + self._debug = settings['DEBUG_SQL_HISTORY_STORE'] + + def open(self, site): + self._store[site] = URLHistory(self._dbinfo) + + def close_site(self, site): + self._store[site].close() + del self._store[site] + + def store(self, site, key, url, parent=None, version=None, post_version=None): + history = self._store[site] + if self._debug: + log.msg("record_version(key=%s, url=%s, parent=%s, version=%s, post_version=%s)" % + (key, url, parent, version, post_version), domain=site, level=log.DEBUG) + history.record_version(key, url, parent, version, post_version) + + def has_site(self, site): + return site in self._store + + def status(self, site, key): + if site in self._store: + history = self._store[site] + return history.get_url_status(key) + + def version_info(self, site, version): + history = self._store[site] + return history.get_version_info(version) diff --git a/scrapy/trunk/scrapy/contrib/item/__init__.py b/scrapy/trunk/scrapy/contrib/item/__init__.py new file mode 100644 index 000000000..3cb63cf0d --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/item/__init__.py @@ -0,0 +1 @@ +from scrapy.contrib.item.models import RobustScrapedItem, ValidationError, ValidationPipeline diff --git a/scrapy/trunk/scrapy/contrib/item/models.py b/scrapy/trunk/scrapy/contrib/item/models.py new file mode 100644 index 000000000..7d6467e93 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/item/models.py @@ -0,0 +1,140 @@ +""" +This module contains some extra base models for scraped items which could be +useful in some Scrapy implementations +""" + +import sha + +from scrapy.item import ScrapedItem +from scrapy.core.exceptions import UsageError, DropItem + +class ValidationError(DropItem): + """Indicates a data validation error""" + def __init__(self,problem,value=None): + self.problem = problem + self.value = value + + def __str__(self): + if self.value is not None: + return '%s "%s"' % (self.problem, self.value) + else: + return '%s' % (self.problem) + +class ValidationPipeline(object): + def process_item(self, domain, response, item): + item.validate() + return item + +class RobustScrapedItem(ScrapedItem): + """ + A more robust scraped item class with a built-in validation mechanism and + minimal versioning support + """ + + ATTRIBUTES = { + 'guid': basestring, # a global unique identifier + 'url': basestring, # the main URL where this item was scraped from + } + + def __init__(self, data=None): + """ + A scraped item can be initialised with a dictionary that will be + squirted directly into the object. + """ + if isinstance(data, dict): + for attr, value in data.iteritems(): + setattr(self, attr, value) + elif data is not None: + raise UsageError("Initialize with dict, not %s" % data.__class__.__name__) + + self.__dict__['_version'] = None + + def __getattr__(self, attr): + # Return None for valid attributes not set, raise AttributeError for invalid attributes + # Note that this method is called only when the attribute is not found in + # self.__dict__ or the class/instance methods. + if attr in self.ATTRIBUTES: + return None + else: + raise AttributeError(attr) + + def __setattr__(self, attr, value): + """ + Set an attribute checking it matches the attribute type declared in self.ATTRIBUTES + """ + if not attr.startswith('_') and attr not in self.ATTRIBUTES: + raise AttributeError('Attribute "%s" is not a valid attribute name. You must add it to %s.ATTRIBUTES' % (attr, self.__class__.__name__)) + + if value is None: + self.__dict__.pop(attr, None) + return + + type1 = self.ATTRIBUTES[attr] + if hasattr(type1, '__iter__'): + if not hasattr(value, '__iter__'): + raise TypeError('Attribute "%s" must be a sequence' % attr) + type2 = type1[0] + for i in value: + if not isinstance(i, type2): + raise TypeError('Attribute "%s" cannot contain %s, only %s' % (attr, i.__class__.__name__, type2.__name__)) + else: + if not isinstance(value, type1): + raise TypeError('Attribute "%s" must be %s, not %s' % (attr, type1.__name__, value.__class__.__name__)) + + self.__dict__[attr] = value + self.__dict__['_version'] = None + + def __delattr__(self, attr): + """ + Delete an attribute from the ScrapedItem instance if it exists. + If not, raise an AttributeError. + """ + if attr in self.__dict__: + del self.__dict__[attr] + self.__dict__['_version'] = None + else: + raise AttributeError("Attribute '%s' doesn't exist" % attr) + + def __eq__(self, other): + return self.version == other.version + + def __ne__(self, other): + return self.version != other.version + + def __repr__(self): + # Generate this format so that it can be deserialized easily: + # ClassName({...}) + reprdict = {} + for k, v in self.__dict__.iteritems(): + if not k.startswith('_'): + reprdict[k] = v + return "%s(%s)" % (self.__class__.__name__, repr(reprdict)) + + def __str__(self) : + return "%s: GUID=%s, url=%s" % ( self.__class__.__name__ , self.guid, self.url ) + + def validate(self): + """Method used to validate item attributes data""" + if not self.guid: + raise ValidationError('A guid is required') + + def copy(self): + """Create a new ScrapedItem object based on the current one""" + import copy + return copy.deepcopy(self) + + @property + def version(self): + """ + Return a (cached) 40 char hash of all the item attributes. + + WARNING: This cached version won't work if mutable products are + modified directly like: + + item.features.append('feature') + """ + if self._version: + return self._version + hash_ = sha.new() + hash_.update("".join(["".join([n, str(v)]) for n,v in sorted(self.__dict__.iteritems())])) + return hash_.hexdigest() diff --git a/scrapy/trunk/scrapy/contrib/memdebug.py b/scrapy/trunk/scrapy/contrib/memdebug.py new file mode 100644 index 000000000..d84fcc3ed --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/memdebug.py @@ -0,0 +1,69 @@ +import pprint +import gc +import socket + +import libxml2 +from pydispatch import dispatcher + +from scrapy.core import signals +from scrapy.core.exceptions import NotConfigured +from scrapy.core.mail import MailSender +from scrapy.stats import stats +from scrapy.extension import extensions +from scrapy.conf import settings + +class MemoryDebugger(object): + + def __init__(self): + if not settings.getbool('MEMDEBUG_ENABLED'): + raise NotConfigured + + self.mail = MailSender() + self.rcpts = settings.getlist('MEMDEBUG_NOTIFY') + + self.domains_scraped = [] + dispatcher.connect(self.domain_opened, signals.domain_opened) + dispatcher.connect(self.engine_started, signals.engine_started) + dispatcher.connect(self.engine_stopped, signals.engine_stopped) + + def engine_started(self): + libxml2.debugMemory(1) + + def engine_stopped(self): + figures = self.collect_figures() + report = self.create_report(figures) + self.print_or_send_report(report) + + def collect_figures(self): + libxml2.cleanupParser() + gc.collect() + + figures = [] + if 'MemoryUsage' in extensions.enabled: + memusage = extensions.enabled['MemoryUsage'] + memusage.update() + figures.append(("Memory usage at startup", int(memusage.data['startup']/1024/1024), "Mb")) + figures.append(("Maximum memory usage", int(memusage.data['max']/1024/1024), "Mb")) + figures.append(("Memory usage at shutdown", int(memusage.virtual/1024/1024), "Mb")) + figures.append(("Objects in gc.garbage", len(gc.garbage), "")) + figures.append(("libxml2 memory leak", libxml2.debugMemory(1), "bytes")) + return figures + + def create_report(self, figures): + s = "" + s += "SCRAPY MEMORY DEBUGGER RESULTS\n\n" + for f in figures: + s += "%-30s : %s %s\n" % f + s += "\n" + if stats: + s += "SCRAPING STATS --------------------------------------------------\n\n" + s += pprint.pformat(stats) + return s + + def print_or_send_report(self, report): + if self.rcpts: + self.mail.send(self.rcpts, "Scrapy Memory Debugger results at %s" % socket.gethostname(), report) + print report + + def domain_opened(self, domain): + self.domains_scraped.append(domain) diff --git a/scrapy/trunk/scrapy/contrib/memusage.py b/scrapy/trunk/scrapy/contrib/memusage.py new file mode 100644 index 000000000..2d0c2aa86 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/memusage.py @@ -0,0 +1,121 @@ +import sys +import os +import pprint +import socket + +from pydispatch import dispatcher + +from scrapy.core import log, signals +from scrapy.core.manager import scrapymanager +from scrapy.core.engine import scrapyengine +from scrapy.core.exceptions import NotConfigured +from scrapy.core.mail import MailSender +from scrapy.stats import stats +from scrapy.conf import settings + +class MemoryUsage(object): + + _proc_status = '/proc/%d/status' % os.getpid() + _scale = {'kB': 1024.0, 'mB': 1024.0*1024.0, + 'KB': 1024.0, 'MB': 1024.0*1024.0} + + def __init__(self): + if not settings.getbool('MEMUSAGE_ENABLED'): + raise NotConfigured + if sys.platform != 'linux2': + raise NotConfigured("MemoryUsage extension is only available on Linux") + + self.warned = False + + self.data = {} + self.data['startup'] = 0 + self.data['max'] = 0 + + scrapyengine.addtask(self.update, 60.0, now=True) + + self.notify_mails = settings.getlist('MEMUSAGE_NOTIFY') + self.limit = settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024 + self.warning = settings.getint('MEMUSAGE_WARNING_MB')*1024*1024 + self.report = settings.getbool('MEMUSAGE_REPORT') + + if self.limit: + scrapyengine.addtask(self._check_limit, 60.0, now=True) + if self.warning: + scrapyengine.addtask(self._check_warning, 60.0, now=True) + + self.mail = MailSender() + + dispatcher.connect(self.engine_started, signal=signals.engine_started) + + + @property + def virtual(self): + return self._vmvalue('VmSize:') + + @property + def resident(self): + return self._vmvalue('VmRSS:') + + @property + def stacksize(self): + return self._vmvalue('VmStk:') + + def engine_started(self): + self.data['startup'] = self.virtual + + def update(self): + if self.virtual > self.data['max']: + self.data['max'] = self.virtual + + def _vmvalue(self, VmKey): + # get pseudo file /proc//status + try: + t = open(self._proc_status) + v = t.read() + t.close() + except: + return 0.0 # non-Linux? + # get VmKey line e.g. 'VmRSS: 9999 kB\n ...' + i = v.index(VmKey) + v = v[i:].split(None, 3) # whitespace + if len(v) < 3: + return 0.0 # invalid format? + # convert Vm value to bytes + return float(v[1]) * self._scale[v[2]] + + def _check_limit(self): + if self.virtual > self.limit: + mem = self.limit/1024/1024 + log.msg("Memory usage exceeded %dM. Shutting down Scrapy..." % mem, level=log.ERROR) + if self.notify_mails: + subj = "%s terminated: memory usage exceeded %dM at %s" % (settings['BOT_NAME'], mem, socket.gethostname()) + self._send_report(self.notify_mails, subj) + scrapymanager.stop() + + def _check_warning(self): + if self.warned: # warn only once + return + if self.virtual > self.warning: + mem = self.warning/1024/1024 + log.msg("Memory usage reached %dM" % mem, level=log.WARNING) + if self.notify_mails: + subj = "%s warning: memory usage reached %dM at %s" % (settings['BOT_NAME'], mem, socket.gethostname()) + self._send_report(self.notify_mails, subj) + self.warned = True + + def _send_report(self, rcpts, subject): + """send notification mail with some additional useful info""" + s = "Memory usage at engine startup : %dM\r\n" % (self.data['startup']/1024/1024) + s += "Maximum memory usage : %dM\r\n" % (self.data['max']/1024/1024) + s += "Current memory usage : %dM\r\n" % (self.virtual/1024/1024) + + s += "ENGINE STATUS ------------------------------------------------------- \r\n" + s += "\r\n" + s += scrapyengine.getstatus() + s += "\r\n" + + if stats: + s += "SCRAPING STATS ------------------------------------------------------ \r\n" + s += "\r\n" + s += pprint.pformat(stats) + self.mail.send(rcpts, subject, s) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/__init__.py b/scrapy/trunk/scrapy/contrib/pbcluster/__init__.py new file mode 100644 index 000000000..40aa182de --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pbcluster/__init__.py @@ -0,0 +1,2 @@ +from scrapy.contrib.pbcluster.worker.manager import ClusterWorker +from scrapy.contrib.pbcluster.master.web import ClusterMasterWeb \ No newline at end of file diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/__init__.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py new file mode 100644 index 000000000..102f35e10 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/manager.py @@ -0,0 +1,220 @@ +import urlparse +import urllib +import bisect +import sys + +from pydispatch import dispatcher + +from twisted.spread import pb +from twisted.internet import reactor +from twisted.python import util + +from scrapy.core import log, signals +from scrapy.core.engine import scrapyengine +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings + +priorities = { 20:'NORMAL', + 10:'QUICK', + 0:'NOW', +} + +# Set priorities module attributes +for val, attr in priorities.items(): + setattr(sys.modules[__name__], "PRIORITY_%s" % attr, val ) + +class Node: + def __init__(self, remote, status, name): + self.__remote = remote + self._set_status(status) + self.name = name + + def _set_status(self, status): + if not status: + self.available = False + else: + self.available = True + self.running = status['running'] + self.pending = status['pending'] + self.maxproc = status['maxproc'] + self.starttime = status['starttime'] + self.timestamp = status['timestamp'] + self.loadavg = status['loadavg'] + self.logdir = status['logdir'] + self.lastcallresponse = status['callresponse'] + + def _remote_call(self, function, *args): + try: + deferred = self.__remote.callRemote(function, *args) + except pb.DeadReferenceError: + self._set_status(None) + log.msg("Lost connection to node %s." % (self.name), log.ERROR) + else: + deferred.addCallbacks(callback=self._set_status, errback=lambda reason: log.msg(reason, log.ERROR)) + + def get_status(self): + self._remote_call("status") + + def schedule(self, domains, spider_settings=None, priority=PRIORITY_NORMAL): + self._remote_call("schedule", domains, spider_settings, priority) + + def stop(self, domains): + self._remote_call("stop", domains) + + def remove(self, domains): + self._remote_call("remove", domains) + +class ClusterMaster(object): + + def __init__(self): + if not settings.getbool('CLUSTER_MASTER_ENABLED'): + raise NotConfigured + self.nodes = {} + self.queue = [] + dispatcher.connect(self._engine_started, signal=signals.engine_started) + + def load_nodes(self): + + def _make_callback(_factory, _name, _url): + + def _errback(_reason): + log.msg("Could not get remote node %s in %s: %s." % (_name, _url, _reason), log.ERROR) + + d = _factory.getRootObject() + d.addCallbacks(callback=lambda obj: self.add_node(obj, _name), errback=_errback) + + """Loads nodes from the CLUSTER_MASTER_NODES setting""" + + for name, url in settings.get('CLUSTER_MASTER_NODES', {}).iteritems(): + if name not in self.nodes: + server, port = url.split(":") + port = eval(port) + log.msg("Connecting to cluster worker %s..." % name) + log.msg("Server: %s, Port: %s" % (server, port)) + factory = pb.PBClientFactory() + try: + reactor.connectTCP(server, port, factory) + except Exception, err: + log.msg("Could not connect to node %s in %s: %s." % (name, url, reason), log.ERROR) + else: + _make_callback(factory, name, url) + + def update_nodes(self): + for node in self.nodes.itervalues(): + node.get_status() + + def add_node(self, cworker, name): + """Add node given its node""" + node = Node(cworker, None, name) + node.get_status() + self.nodes[name] = node + log.msg("Added cluster worker %s" % name) + + def remove_node(self, nodename): + raise NotImplemented + + def schedule(self, domains, spider_settings=None, nodename=None, priority=PRIORITY_NORMAL): + if nodename: + self.nodes[nodename].schedule(domains, spider_settings, priority) + else: + self._dispatch_domains(domains, spider_settings, priority) + + def stop(self, domains): + to_stop = {} + for domain in domains: + node = self.running.get(domain, None) + if node: + if node.name not in to_stop: + to_stop[node.name] = [] + to_stop[node.name].append(domain) + + for nodename, domains in to_stop.iteritems(): + self.nodes[nodename].stop(domains) + + def remove(self, domains): + to_remove = {} + for domain in domains: + node = self.pending.get(domain, None) + if node: + if node.name not in to_remove: + to_remove[node.name] = [] + to_remove[node.name].append(domain) + + for nodename, domains in to_remove.iteritems(): + self.nodes[nodename].remove(domains) + + def discard(self, domains): + """Stop and remove all running and pending instances of the given + domains""" + self.remove(domains) + self.stop(domains) + + @property + def running(self): + """Return dict of running domains as domain -> node""" + d = {} + for node in self.nodes.itervalues(): + for proc in node.running: + d[proc['domain']] = node + return d + + @property + def pending(self): + """Return dict of pending domains as domain -> node""" + d = {} + for node in self.nodes.itervalues(): + for p in node.pending: + d[p['domain']] = node + return d + + @property + def available_nodes(self): + return (node for node in self.nodes.itervalues() if node.available) + + def _dispatch_domains(self, domains, spider_settings, priority): + """Schedule the given domains in the availables nodes as good as + possible. The algorithm follows the next rules (in order): + + 1. search for nodes with available capacity(running < maxproc) and (if + any) schedules the domains there + + 2. if there isn't any node with available capacity it schedules the + domain in the node with the smallest number of pending spiders + """ + + to_schedule = {} # domains to schedule per node + pending_node = [] # list of #pending, node + + for node in self.available_nodes: + capacity = node.maxproc - len(node.running) + #order nodes in pending_node according to insertion position, calculated from priority comparison, for stage 2. + i = 0 + for p in node.pending: + if p['priority'] <= priority: + i += 1 + else: + break + bisect.insort(pending_node, (i, node)) + + #stage 1: use available capacity + to_schedule[node.name] = [] + while domains and capacity > 0: + to_schedule[node.name].append(domains.pop(0)) + capacity -= 1 + if not domains: + break + + #stage 2: queue in pendings the remaining domains. + # a) pops out minor insertion-point node b) schedules the domain c) reinserts the node in list with insertion-point incremented by one. + for domain in domains: + insert_point, node = pending_node.pop(0) + to_schedule[node.name].append(domain) + bisect.insort(pending_node, (insert_point+1, node)) + + for nodename, domains in to_schedule.iteritems(): + if domains: + self.nodes[nodename].schedule(domains, spider_settings, priority) + + def _engine_started(self): + self.load_nodes() + scrapyengine.addtask(self.update_nodes, settings.getint('CLUSTER_MASTER_POLL_INTERVAL')) diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py new file mode 100644 index 000000000..7306f7b03 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pbcluster/master/web.py @@ -0,0 +1,174 @@ +import datetime + +from pydispatch import dispatcher + +from scrapy.spider import spiders +from scrapy.management.web import banner, webconsole_discover_module +from scrapy.contrib.pbcluster.master.manager import ClusterMaster, priorities + +class ClusterMasterWeb(ClusterMaster): + webconsole_id = 'cluster_master' + webconsole_name = 'Cluster master' + + def __init__(self): + ClusterMaster.__init__(self) + + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def webconsole_render(self, wc_request): + changes = "" + if wc_request.path == '/cluster_master/nodes/': + return self.render_nodes(wc_request) + elif wc_request.path == '/cluster_master/domains/': + return self.render_domains(wc_request) + elif wc_request.args: + changes = self.webconsole_control(wc_request) + + s = self.render_header() + + s += "

    Home

    \n" + + s += "\n" + s += "\n" + for node in self.nodes.itervalues(): + #chkbox = "" % domain if node.status in ["up", "idle"] else " " + nodelink = "%s" % (node.name, node.name) + chkbox = " " + loadavg = "%.2f %.2f %.2f" % node.loadavg + s += "\n" % \ + (chkbox, nodelink, node.available, len(node.running), node.maxproc, len(node.pending), loadavg) + s += "
     NameAvailableRunningPendingLoad.avg
    %s%s%s%d/%d%d%s
    \n" + + s += "\n" + s += "\n" + + return str(s) + + def webconsole_control(self, wc_request): + args = wc_request.args + + if "updatenodes" in args: + self.update_nodes() + + if "schedule" in args: + node = args["node"][0] if "node" in args else None + self.schedule(args["schedule"], nodename=node, priority=eval(args["priority"][0])) + + if "stop" in args: + self.stop(args["stop"]) + + if "remove" in args: + self.remove(args["remove"]) + + return "" + + def render_nodes(self, wc_request): + if wc_request.args: + self.webconsole_control(wc_request) + + now = datetime.datetime.utcnow() + + s = self.render_header() + for node in self.nodes.itervalues(): + if node.available: + s += "

    %s

    \n" % (node.name, node.name) + + s += "

    Running domains

    \n" + if node.running: + s += "
    \n" + s += "\n" + s += "\n" + for proc in node.running: + chkbox = "" % proc['domain'] if proc['status'] == "running" else " " + start_time = proc.get('starttime', None) + elapsed = now - start_time if start_time else None + s += "\n" % \ + (chkbox, proc['pid'], proc['domain'], proc['status'], elapsed, proc['logfile']) + s += "
     PIDDomainStatusRunning timeLog file
    %s%s%s%s%s%s
    \n" + s += "\n" % node.name + s += "

    \n" % node.name + s += "
    \n" + else: + s += "

    No running domains on %s

    \n" % node.name + + # pending domains + s += "

    Pending domains

    \n" + if node.pending: + s += "
    \n" + s += "\n" + s += "\n" % node.name + s += "

    \n" % node.name + s += "
    \n" + else: + s += "

    No pending domains on %s

    \n" % node.name + + return str(s) + + def render_domains(self, wc_request): + if wc_request.args: + self.webconsole_control(wc_request) + + enabled_domains = set(spiders.asdict(include_disabled=False).keys()) + print "Enabled domains: %s" % len(enabled_domains) + inactive_domains = enabled_domains - set(self.running.keys() + self.pending.keys()) + + s = self.render_header() + + s += "

    Schedule domains

    \n" + + s += "Inactive domains (not running or pending)
    " + s += "
    \n" + s += "\n" + s += "
    \n" + s += "Node (only available nodes shown):
    \n" + s += "
    \n" + s += "Priority:
    \n" + s += "\n" + s += "

    \n" + s += "
    \n" + + s += "

    Domains

    \n" + + s += "\n" + s += "\n" + s += self._domains_table(self.running, 'running') + s += self._domains_table(self.pending, 'pending') + s += "
    DomainStatusNode
    \n" + + return str(s) + + def render_header(self): + s = banner(self) + s += "

    Nav: " + s += "Home | " + s += "Domains | " + s += "Nodes (update)" + s += "

    " + return s + + def _domains_table(self, dict_, status): + s = "" + for domain, node in dict_.iteritems(): + s += "%s%s%s\n" % (domain, status, node.name) + return s + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/__init__.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py new file mode 100644 index 000000000..8aa208f71 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pbcluster/worker/manager.py @@ -0,0 +1,151 @@ +import sys +import os +import time +import datetime + +from twisted.internet import protocol, reactor +from twisted.spread import pb + +from scrapy.core import log +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings +from scrapy.core.engine import scrapyengine + +class ScrapyProcessProtocol(protocol.ProcessProtocol): + def __init__(self, procman, domain, logfile=None, spider_settings=None): + self.procman = procman + self.domain = domain + self.logfile = logfile + self.start_time = datetime.datetime.utcnow() + self.status = "starting" + self.pid = -1 + + env = {'SCRAPY_LOGFILE': self.logfile, 'SCRAPY_CLUSTER_WORKER_ENABLED': '0', 'SCRAPY_WEBCONSOLE_ENABLED': '0'} + #We conserve original setting format for info purposes (avoid lots of unnecesary "SCRAPY_") + self.settings = spider_settings or {} + for k in self.settings: + env["SCRAPY_%s" % k] = self.settings[k] + self.env = env + + def __str__(self): + return "" % (self.domain, self.pid, self.status) + + def as_dict(self): + return {"domain": self.domain, "pid": self.pid, "status": self.status, "settings": self.settings, "logfile": self.logfile, "starttime": self.start_time} + + def connectionMade(self): + self.pid = self.transport.pid + log.msg("ClusterWorker: started domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile)) + self.transport.closeStdin() + self.status = "running" + + def processEnded(self, status_object): + log.msg("ClusterWorker: finished domain=%s, pid=%d, log=%s" % (self.domain, self.pid, self.logfile)) + del self.procman.running[self.domain] + self.procman.next_pending() + +class ClusterWorker(pb.Root): + + def __init__(self): + if not settings.getbool('CLUSTER_WORKER_ENABLED'): + raise NotConfigured + + self.maxproc = settings.getint('CLUSTER_WORKER_MAXPROC') + self.logdir = settings['CLUSTER_WORKER_LOGDIR'] + self.running = {} + self.pending = [] + self.starttime = time.time() + port = settings.getint('CLUSTER_WORKER_PORT') + scrapyengine.listenTCP(port, pb.PBServerFactory(self)) + + def remote_schedule(self, domains, spider_settings=None, priority=20): + """Schedule new domains to be crawled in a separate processes""" + + responses = [] + for domain in domains: + if len(self.running) < self.maxproc and domain not in self.running: + self._run(domain, spider_settings) + responses.append("Started %s" % self.running[domain]) + else: + i = 0 + for p in self.pending: + if p['priority'] <= priority: + i += 1 + else: + break + self.pending.insert(i, {'domain': domain, 'settings': spider_settings, 'priority': priority}) + responses.append("Scheduled domain %s at position %s in queue" % (domain, i)) + return self.status(responses) + + def remote_stop(self, domains): + """Stop running domains. For removing pending (not yet started) domains + use remove() instead""" + + responses = [] + for domain in domains: + if domain in self.running: + proc = self.running[domain] + log.msg("ClusterWorker: Sending shutdown signal to domain=%s, pid=%d" % (domain, proc.pid)) + proc.transport.signalProcess('INT') + proc.status = "closing" + responses.append("Stopped process %s" % proc) + else: + responses.append("%s: domain not running." % domain) + return self.status(responses) + + def remote_remove(self, domains): + """Remove all scheduled instances of the given domains (if it hasn't + started yet). Otherwise use stop()""" + + responses = [] + for domain in domains: + to_remove = [] + for p in self.pending: + if p['domain'] == domain: + to_remove.append(p) + + for p in to_remove: + self.pending.remove(p) + responses.append("Unscheduled domain %s" % domain) + return self.status(responses) + + def remote_status(self): + return self.status() + + def status(self, response="Status Response"): + status = {} + status["pending"] = self.pending + status["running"] = [ self.running[k].as_dict() for k in self.running.keys() ] + status["starttime"] = self.starttime + status["timestamp"] = time.time() + status["maxproc"] = self.maxproc + status["loadavg"] = os.getloadavg() + status["logdir"] = self.logdir + status["callresponse"] = response + return status + + def next_pending(self): + """Run the next domain in the pending list, which is not already running""" + + if len(self.running) >= self.maxproc: + return + for p in self.pending: + if p['domain'] not in self.running: + self._run(p['domain'], p['settings']) + self.pending.remove(p) + return + + def _run(self, domain, spider_settings=None): + """Spawn process to run the given domain. Don't call this method + directly. Instead use schedule().""" + + logfile = os.path.join(self.logdir, domain, time.strftime("%FT%T.log")) + if not os.path.exists(os.path.dirname(logfile)): + os.makedirs(os.path.dirname(logfile)) + scrapy_proc = ScrapyProcessProtocol(self, domain, logfile, spider_settings) + + args = [sys.executable, sys.argv[0], 'crawl', domain] + proc = reactor.spawnProcess(scrapy_proc, sys.executable, args=args, env=scrapy_proc.env) + self.running[domain] = scrapy_proc + + diff --git a/scrapy/trunk/scrapy/contrib/pbcluster/worker/testworker.py b/scrapy/trunk/scrapy/contrib/pbcluster/worker/testworker.py new file mode 100755 index 000000000..23ef4c355 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pbcluster/worker/testworker.py @@ -0,0 +1,29 @@ +#!/usr/bin/python2.5 + +from twisted.spread import pb +from twisted.internet import reactor +from twisted.python import util +import sys + +factory = pb.PBClientFactory() +reactor.connectTCP("localhost", 8789, factory) +d = factory.getRootObject() + +sys.argv.pop(0) + +if not sys.argv: + d.addCallback(lambda object: object.callRemote("status")) +elif sys.argv[0] == "-d": + try: + priority = eval(sys.argv[2]) + except: + priority = 20 + d.addCallback(lambda object: object.callRemote("schedule", [sys.argv[1]], priority=priority)) +elif sys.argv[0] == "-s": + d.addCallback(lambda object: object.callRemote("stop", [sys.argv[1]])) +elif sys.argv[0] == "-r": + d.addCallback(lambda object: object.callRemote("remove", [sys.argv[1]])) + +d.addCallbacks(callback = util.println, errback = lambda reason: 'error: '+str(reason.value)) +d.addCallback(lambda _: reactor.stop()) +reactor.run() diff --git a/scrapy/trunk/scrapy/contrib/pipeline/__init__.py b/scrapy/trunk/scrapy/contrib/pipeline/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py b/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py new file mode 100644 index 000000000..e4050bf88 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pipeline/shoveitem.py @@ -0,0 +1,37 @@ +""" +A pipeline to persist objects using shove. + +New is a "new generation" shelve. For more information see: +http://pypi.python.org/pypi/shove +""" + +from string import Template + +from shove import Shove +from pydispatch import dispatcher + +from scrapy.core import signals +from scrapy.conf import settings + +class ShoveItemPipeline(object): + + def __init__(self): + self.uritpl = settings['SHOVEITEM_STORE_URI'] + if not self.uritpl: + raise NotConfigured + self.opts = settings['SHOVEITEM_STORE_OPT'] or {} + self.stores = {} + + dispatcher.connect(self.domain_open, signal=signals.domain_open) + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + + def process_item(self, domain, response, item): + self.stores[domain][str(item.guid)] = item + return item + + def domain_open(self, domain): + uri = Template(self.uritpl).substitute(domain=domain) + self.stores[domain] = Shove(uri, **self.opts) + + def domain_closed(self, domain): + self.stores[domain].sync() diff --git a/scrapy/trunk/scrapy/contrib/pipeline/show.py b/scrapy/trunk/scrapy/contrib/pipeline/show.py new file mode 100644 index 000000000..3edde2bea --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/pipeline/show.py @@ -0,0 +1,15 @@ +""" +Pipeline to print Items +""" +from scrapy.core import log +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings + +class ShowItemPipeline(object): + def __init__(self): + if not settings['DEBUG_SHOWITEM']: + raise NotConfigured + + def process_item(self, domain, response, item): + log.msg("Scraped: \n%s" % repr(item), log.DEBUG, domain=domain) + return item diff --git a/scrapy/trunk/scrapy/contrib/prioritizers.py b/scrapy/trunk/scrapy/contrib/prioritizers.py new file mode 100644 index 000000000..e23a6c74a --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/prioritizers.py @@ -0,0 +1,34 @@ +import time + +from scrapy.core.exceptions import NotConfigured +from scrapy.store.db import DomainDataHistory +from scrapy.conf import settings + +class LessScrapedPrioritizer(object): + """ + A spider prioritizer based on these few simple rules: + 1. if spider was never scraped, it has top priority + 2. if spider was scraped before, then the less recently the spider + has been scraped, the more priority it has + """ + def __init__(self, elements): + if not settings['SCRAPING_DB']: + raise NotConfigured("SCRAPING_DB setting is required") + + self.ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') + domains_to_scrape = set(elements) + + self.priorities = {} + + for domain in domains_to_scrape: + stat = self.ddh.getlast(domain, path="start_time") + if stat and stat[1]: + last_started = stat[1] + # spider is the timestamp of last start time + self.priorities[domain] = time.mktime(last_started.timetuple()) + else: + # if domain was never scraped, it has top priority + self.priorities[domain] = 1 + + def get_priority(self, element): + return self.priorities[element] diff --git a/scrapy/trunk/scrapy/contrib/response/__init__.py b/scrapy/trunk/scrapy/contrib/response/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/response/soup.py b/scrapy/trunk/scrapy/contrib/response/soup.py new file mode 100644 index 000000000..9708b21fe --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/response/soup.py @@ -0,0 +1,26 @@ +""" +The ResponseSoup extension causes the Response objects to grow a new method +("getsoup") which returns a (cached) BeautifulSoup object of its body, and a +"soup" attribute with the same effect. The soup argument is provided for +convenience, but you cannot pass any BeautifulSoup constructor arguments (which +you can do with the getsoup() method). + +For more information about BeautifulSoup see: +http://www.crummy.com/software/BeautifulSoup/documentation.html +""" + +from BeautifulSoup import BeautifulSoup + +from scrapy.http import Response + + +class ResponseSoup(object): + def __init__(self): + setattr(Response, 'getsoup', getsoup) + setattr(Response, 'soup', property(getsoup)) + +def getsoup(response, **kwargs): + if not hasattr(response, '_soup'): + body = response.body.to_string() if response.body is not None else "" + setattr(response, '_soup', BeautifulSoup(body, **kwargs)) + return response._soup diff --git a/scrapy/trunk/scrapy/contrib/spider/__init__.py b/scrapy/trunk/scrapy/contrib/spider/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/spider/profiler.py b/scrapy/trunk/scrapy/contrib/spider/profiler.py new file mode 100644 index 000000000..2c8638bd7 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spider/profiler.py @@ -0,0 +1,60 @@ +""" +SpiderProfiler is an extension that hooks itself into every Request callback +returned from spiders to measure the processing time and memory allocation +caused by spiders code. + +The results are collected using the StatsCollector. + +This extension introduces a big impact on crawling performance, so enable only +when needed. +""" + +import datetime + +from pydispatch import dispatcher + +from scrapy.extension import extensions +from scrapy.core import signals +from scrapy.core.exceptions import NotConfigured +from scrapy.stats import stats +from scrapy.conf import settings + +class SpiderProfiler(object): + + def __init__(self): + if not settings.getbool('SPIDERPROFILER_ENABLED'): + raise NotConfigured + dispatcher.connect(self._request_received, signals.request_received) + dispatcher.connect(self._engine_started, signals.engine_started) + + def _engine_started(self): + self.memusage = extensions.enabled.get('MemoryUsage', None) + + def _request_received(self, request, spider): + old_cbs = request.deferred.callbacks[0] + new_cbs = ((self._profiled_callback(old_cbs[0][0], spider), old_cbs[0][1], old_cbs[0][2]), old_cbs[1]) + request.deferred.callbacks[0] = new_cbs + + def _profiled_callback(self, function, spider): + def new_callback(*args, **kwargs): + tbefore = datetime.datetime.now() + mbefore = self._memusage() + r = function(*args, **kwargs) + tafter = datetime.datetime.now() + mafter = self._memusage() + ct = tafter-tbefore + tcc = stats.getpath('%s/profiling/total_callback_time' % spider.domain_name, datetime.timedelta(0)) + sct = stats.getpath('%s/profiling/slowest_callback_time' % spider.domain_name, datetime.timedelta(0)) + stats.setpath('%s/profiling/total_callback_time' % spider.domain_name, tcc+ct) + if ct > sct: + stats.setpath('%s/profiling/slowest_callback_time' % spider.domain_name, ct) + stats.setpath('%s/profiling/slowest_callback_name' % spider.domain_name, function.__name__) + stats.setpath('%s/profiling/slowest_callback_url' % spider.domain_name, args[0].url) + if self.memusage: + tma = stats.getpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, 0) + stats.setpath('%s/profiling/total_mem_allocated_in_callbacks' % spider.domain_name, tma+mafter-mbefore) + return r + return new_callback + + def _memusage(self): + return self.memusage.virtual if self.memusage else 0.0 diff --git a/scrapy/trunk/scrapy/contrib/spider/reloader.py b/scrapy/trunk/scrapy/contrib/spider/reloader.py new file mode 100644 index 000000000..2ff345501 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spider/reloader.py @@ -0,0 +1,18 @@ +""" +Reload spider modules once they are finished scraping + +This is to release any resources held on to by scraping spiders. +""" +import sys +from pydispatch import dispatcher +from scrapy.core import log, signals + +class SpiderReloader(object): + + def __init__(self): + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + + def domain_closed(self, domain, spider): + module = spider.__module__ + log.msg("reloading module %s" % module, domain=domain) + reload(sys.modules[module]) diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py new file mode 100644 index 000000000..f3d4c85d3 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/depth.py @@ -0,0 +1,36 @@ +""" +DepthMiddleware is a scrape middleware used for tracking the depth of each +Request inside the site being scraped. It can be used to limit the maximum +depth to scrape or things like that +""" + +from scrapy.core import log +from scrapy.http import Request +from scrapy.stats import stats +from scrapy.conf import settings + +class DepthMiddleware(object): + + def __init__(self): + self.maxdepth = settings.getint('DEPTH_LIMIT') + self.stats = settings.getbool('DEPTH_STATS') + if self.stats and self.maxdepth: + stats.setpath('_envinfo/request_depth_limit', self.maxdepth) + + def process_result(self, response, result, spider): + def _filter(request): + if isinstance(request, Request): + request.depth = response.request.depth + 1 + if self.maxdepth and request.depth > self.maxdepth: + log.msg("Ignoring link (depth > %d): %s " % (self.maxdepth, request.url), level=log.DEBUG, domain=spider.domain_name) + return False + elif self.stats: + stats.incpath('%s/request_depth_count/%s' % (spider.domain_name, request.depth)) + if request.depth > stats.getpath('%s/request_depth_max' % spider.domain_name, 0): + stats.setpath('%s/request_depth_max' % spider.domain_name, request.depth) + return True + + if self.stats and response.request.depth == 0: # otherwise we loose stats for depth=0 + stats.incpath('%s/request_depth_count/0' % spider.domain_name) + + return [r for r in result or () if _filter(r)] diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py new file mode 100644 index 000000000..20b681461 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/offsite.py @@ -0,0 +1,13 @@ +from scrapy.core import log +from scrapy.http import Request +from scrapy.utils.url import url_is_from_spider + +class OffsiteMiddleware(object): + def process_result(self, response, result, spider): + def _filter(r): + if isinstance(r, Request) and not url_is_from_spider(r.url, spider): + log.msg("Ignoring link (offsite): %s " % r.url, level=log.DEBUG, domain=spider.domain_name) + return False + return True + return [r for r in result or () if _filter(r)] + diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py new file mode 100644 index 000000000..fbaa3d411 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/referer.py @@ -0,0 +1,12 @@ +from scrapy.core import log +from scrapy.http import Request + + +class CrawlMiddleware(object): + def process_result(self, response, result, spider): + def _set_referer(r): + if isinstance(r, Request): + r.headers.setdefault('Referer', response.url) + return r + return [_set_referer(r) for r in result or ()] + diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py new file mode 100644 index 000000000..4e5272529 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/restrict.py @@ -0,0 +1,21 @@ +""" +Scrapemiddlware to restrict crawling to only some particular URLs +""" + +from scrapy.http import Request +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings + +class RestrictMiddleware(object): + def __init__(self): + self.allowed_urls = set(settings.getlist('RESTRICT_TO_URLS')) + if not self.allowed_urls: + raise NotConfigured + + def process_result(self, response, result, spider): + def _filter(r): + if isinstance(r, Request) and r.url not in self.allowed_urls: + return False + return True + return [r for r in result or () if _filter(r)] + diff --git a/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py new file mode 100644 index 000000000..d9658f4a7 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/spidermiddleware/urllength.py @@ -0,0 +1,22 @@ +from scrapy.core import log +from scrapy.http import Request +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings + +class UrlLengthMiddleware(object): + """This middleware discard requests with URLs longer than URLLENGTH_LIMIT""" + + def __init__(self): + self.maxlength = settings.getint('URLLENGTH_LIMIT') + if not self.maxlength: + raise NotConfigured + + def process_result(self, response, result, spider): + def _filter(request): + if isinstance(request, Request) and len(request.url) > self.maxlength: + log.msg("Ignoring link (url length > %d): %s " % (self.maxlength, request.url), level=log.DEBUG, domain=spider.domain_name) + return False + else: + return True + + return [r for r in result or () if _filter(r)] diff --git a/scrapy/trunk/scrapy/contrib/web/__init__.py b/scrapy/trunk/scrapy/contrib/web/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/web/http.py b/scrapy/trunk/scrapy/contrib/web/http.py new file mode 100644 index 000000000..dcc7732b0 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/web/http.py @@ -0,0 +1,115 @@ +"""A django alike request-response model + +most of this code is borrowed from django +""" + +from Cookie import SimpleCookie +from scrapy.utils.datatypes import MultiValueDict, CaselessDict + + +def build_httprequest(twistedrequest): + """Translate twisted request object to a django request approach""" + request = HttpRequest() + request.path = twistedrequest.path + request.method = twistedrequest.method.upper() + request.COOKIES = SimpleCookie(twistedrequest.received_cookies) + request.HEADERS = Headers(twistedrequest.received_headers) + request.ARGS = MultiValueDict(twistedrequest.args) + request.FILES = {} # not yet supported + request.content = twistedrequest.content + request.twistedrequest = twistedrequest + return request + + +class HttpRequest(object): + def __init__(self): + self.path = '' + self.method = None + self.COOKIES = {} + self.HEADERS = {} + self.ARGS = {} + self.FILES = {} + + +class HttpResponse(object): + status_code = 200 + + def __init__(self, content='', status=None, content_type=None): + content_type = content_type or "text/html; charset=utf-8" + self._headers = {'content-type': content_type} + self.content = content + self.cookies = SimpleCookie() + self.status_code = status + + def __str__(self): + "Full HTTP message, including headers" + return '\n'.join(['%s: %s' % (key, value) + for key, value in self._headers.items()]) \ + + '\n\n' + self.content + + def __setitem__(self, header, value): + self._headers[header.lower()] = value + + def __delitem__(self, header): + try: + del self._headers[header.lower()] + except KeyError: + pass + + def __getitem__(self, header): + return self._headers[header.lower()] + + def has_header(self, header): + "Case-insensitive check for a header" + return self._headers.has_key(header.lower()) + + __contains__ = has_header + + def items(self): + return self._headers.items() + + def get(self, header, alternate): + return self._headers.get(header, alternate) + + def set_cookie(self, key, value='', max_age=None, expires=None, path='/', domain=None, secure=None): + self.cookies[key] = value + for var in ('max_age', 'path', 'domain', 'secure', 'expires'): + val = locals()[var] + if val is not None: + self.cookies[key][var.replace('_', '-')] = val + + def delete_cookie(self, key, path='/', domain=None): + self.cookies[key] = '' + if path is not None: + self.cookies[key]['path'] = path + if domain is not None: + self.cookies[key]['domain'] = domain + self.cookies[key]['expires'] = 0 + self.cookies[key]['max-age'] = 0 + + +class Headers(CaselessDict): + def __init__(self, source=None, encoding='utf-8'): + self.encoding = encoding + + if getattr(source, 'iteritems', None): + d = source.iteritems() + else: + d = source # best effort + + # can't use CaselessDict.__init__(self, d) because it doesn't call __setitem__ + for k,v in d: + self.__setitem__(k.lower(), v) + + def normkey(self, key): + return key.title() # 'Content-Type' styles headers + + def __setitem__(self, key, value): + """Headers must not be unicode""" + if isinstance(key, unicode): + key = key.encode(self.encoding) + if isinstance(value, unicode): + value = value.encode(self.encoding) + super(Headers, self).__setitem__(key, value) + + diff --git a/scrapy/trunk/scrapy/contrib/web/json.py b/scrapy/trunk/scrapy/contrib/web/json.py new file mode 100644 index 000000000..6c7aa06fa --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/web/json.py @@ -0,0 +1,129 @@ +from functools import wraps +from twisted.internet import defer +from scrapy.utils.misc import mustbe_deferred + +from .http import HttpResponse + + +JSONCALLBACK_RE = '^[a-zA-Z][a-zA-Z_.-]*$' +JSON_CONTENT_TYPES = ('application/json',) + + +def serialize(obj): + global _serialize + if not _serialize: + _loadserializers() + return _serialize(obj) + +def unserialize(obj): + global _unserialize + if not _unserialize: + _loadserializers() + return _unserialize(obj) + + +class JsonException(Exception): + pass + +class JsonResponse(HttpResponse): + def __init__(self, content=None, callback=None, serialize=serialize, *args, **kwargs): + content = serialize(content) + if callback: # JSONP support + status, content = 200, '%s(%s)' % (callback, content) + kwargs.setdefault('content_type', 'application/x-javascript') + HttpResponse.__init__(self, content=content, *args, **kwargs) + +class JsonResponseAccepted(JsonResponse): + status_code = 202 + +class JsonResponseNoContent(JsonResponse): + status_code = 204 + +class JsonResponseNotModified(JsonResponse): + status_code = 304 + +class JsonResponseBadRequest(JsonResponse): + status_code = 400 + +class JsonResponseUnauthorized(JsonResponse): + status_code = 401 + +class JsonResponseForbidden(JsonResponse): + status_code = 403 + +class JsonResponseNotFound(JsonResponse): + status_code = 404 + +class JsonResponseInternalServerError(JsonResponse): + status_code = 500 + +class JsonResponseNotImplemented(JsonResponse): + status_code = 501 + + +def json(func): + """ Decorator to wrap a json prepared view and return a JsonResponse + + if content-type is application/json, sets request.JSON to unserialized request body. + in case of unserialization failure, returns JsonResponseBadRequest() + + if returned data from func is a dictionary, serialize it and returns JsonResponse() + """ + if not hasattr(func, '__call__'): + raise TypeError('The argument should be a callable') + + @wraps(func) + def wrapper(request, *args, **kwargs): + json_callback = request.ARGS.get('callback') # JSONP support + request.method = method = _x_http_method_override(request) + request.content_type = ct = content_type(request) + request.JSON = None + + if method in ('POST', 'PUT'): + if ct in JSON_CONTENT_TYPES: + body = request.content.read() + try: + request.JSON = unserialize(body) + except Exception, e: + return JsonResponseBadRequest('Invalid json: %s' % e ) + + def _onsuccess(response): + if not isinstance(response, HttpResponse): + return JsonResponse(response, json_callback) # best effort + return response + + ret = mustbe_deferred(func, request, *args, **kwargs) + ret.addCallback(_onsuccess) + return ret + return wrapper + +def content_type(request): + ct = request.HEADERS.get('content-type','') + return ct.split(';')[0].strip() + +def _x_http_method_override(request): + """ support for X-Http-Method-Override hack + + some clients does not support methods others than GET and POST, that clients + has a chance to set an extra header to indicate intended method. + """ + return request.HEADERS.get('x-http-method-override', request.method).upper() + + +## json serializers + +_serialize = _unserialize = None +def _loadserializers(): + global _serialize, _unserialize + try: + import cjson + _serialize = cjson.encode + _unserialize = cjson.decode + except ImportError: + try: + import simplejson + _serialize = simplejson.dumps + _unserialize = simplejson.loads + except ImportError: + assert 0, 'json serialization needs cjson or simplejson modules' + diff --git a/scrapy/trunk/scrapy/contrib/web/service.py b/scrapy/trunk/scrapy/contrib/web/service.py new file mode 100644 index 000000000..db112d5af --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/web/service.py @@ -0,0 +1,90 @@ +import re +import urllib +import simplejson +from sha import sha +from twisted.internet import defer + +from scrapy.core.engine import scrapyengine +from scrapy.spider import spiders +from scrapy.http import Request +from scrapy.item import ScrapedItem +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings +from scrapy.utils.misc import memoize +from lrucache import LRUCache + +from .site import WebSite, WebResource +from .http import HttpResponse +from .json import JsonResponse + +JSONCALLBACK_RE = '^[a-zA-Z][a-zA-Z_.-]*$' +CACHESIZE = settings.get('WS_CACHESIZE', 20) + + +def _urlhash(request): + h = sha() + for a in sorted(request.ARGS): + h.update(request.ARGS[a]) + return h.hexdigest() + + +@memoize(cache=LRUCache(CACHESIZE), hash=_urlhash) +def url_to_guid(httprequest): + url = httprequest.ARGS.get('url') + if not url: + return HttpResponse('Bad Request', 400) + url = urllib.unquote(url) + + jsoncb = httprequest.ARGS.get('callback') + if jsoncb and not re.match(JSONCALLBACK_RE, jsoncb): + return HttpResponse('Bad callback argument', 400) + + def _response(guids=(), message=None): + content = { + 'guids': list(guids), + 'domain': getattr(spider, 'domain_name', None), + 'message': message, + } + return JsonResponse(content=content, callback=jsoncb) + + spider = spiders.fromurl(url) + if not spider: + return _response(message='No crawler found for site') + + if httprequest.ARGS.get('dontcrawl'): + return _response() + + + def _on_error(_failure): + return _response(message='Error downloading url from site') + + def _on_success(pagedata): + try: + items = spider.identify(pagedata) + except Exception, ex: + return _response(message='Error processing url') + + guids = [i.guid for i in items if isinstance(i, ScrapedItem)] + return _response(guids=guids) + + deferred = defer.Deferred().addCallbacks(_on_success, _on_error) + request = Request(url=url, callback=deferred, dont_filter=True) + schd = scrapyengine.schedule(request, spider) + schd.chainDeferred(deferred) + return deferred + + +urlmapping = ( + ('^ws/tools/url_to_guid/$', url_to_guid), +) + + +class UrlToGuidService(WebSite): + def __init__(self): + if not settings['WS_ENABLED']: + raise NotConfigured + + port = settings.getint('WS_PORT') or 8088 + timeout = settings.getint('WS_TIMEOUT') or 15 # seconds + resource = WebResource(urlmapping, timeout=timeout) + WebSite.__init__(self, port=port, resource=resource) diff --git a/scrapy/trunk/scrapy/contrib/web/site.py b/scrapy/trunk/scrapy/contrib/web/site.py new file mode 100644 index 000000000..5ba47ff1c --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/web/site.py @@ -0,0 +1,133 @@ +"""Twisted website object as django + +################################################################################ +## Simple Usage example: + +from twisted.internet import reactor +from scrapy.contrib.web.http import WebSite, HttpResponse + +def helloword(request): + return HttpResponse('Hello World!') + +def hello(request, name): + return HttpResponse('Hello %s' % name) + +urls = ( + ('^hello/(?P\w+)/$', hello), + ('^$', helloword), + ) + + +resource = WebResource(urls) +site = WebSite(port=8081, resource=resource) +reactor.run() + +# now go to http://localhost:8081/ + + + +################################################################################ +## Complex usage example: + +from twisted.internet import reactor, defer +from scrapy.contrib.web.http import WebSite, HttpResponse + +def delayed(request): + def _callback(result): + return HttpResponse('Heavy task completed: %s' % result) + + def _errback(_failure): + return HttpResponse('Internal Server Error: %s' % _failure, status=500) + + def heavytask(_): + import random + assert random.randint(0,1), "Exception found processing request" + return _ + + d = defer.Deferred().addCallback(heavytask) + d.addCallbacks(_callback, _errback) + reactor.callLater(1, d.callback, "Well done") + return d + +urls = (('^delayed/$', delayed),) + +resource = WebResource(urls) +site = WebSite(port=8081, resource=resource) +reactor.run() + +""" + +import re + +from twisted.web import server, resource +from twisted.internet import reactor +from scrapy.utils.misc import mustbe_deferred + +from .http import HttpResponse, build_httprequest + + +def urlresolver(urls, path): + """Simple path to view mapper""" + path = path.lstrip('/') + for pathre, view in urls: + m = re.search(pathre, path) + if m: + kwargs = m.groupdict() + args = () if kwargs else m.groups() + return view, args, kwargs + return None, (), {} + + +class WebSite(server.Site): + def __init__(self, port=None, *args, **kwargs): + server.Site.__init__(self, *args, **kwargs) + if port: + self.bind(port) + + def bind(self, port): + from scrapy.core.engine import scrapyengine + scrapyengine.listenTCP(port, self) + + +class WebResource(resource.Resource): + """Translate twisted web approach to django alike way""" + isLeaf = True + debug = True + + def __init__(self, urls, timeout=3, urlresolver=urlresolver): + resource.Resource.__init__(self) + self.urlresolver = urlresolver + self.timeout = timeout + self.urls = urls + + def render(self, twistedrequest): + httprequest = build_httprequest(twistedrequest) + + def _send_response(response): + assert isinstance(response, HttpResponse), 'view should return a HttpResponse object' + twistedrequest.setResponseCode(response.status_code or 200) + for key, val in response.items(): + twistedrequest.setHeader(key, response[key]) + twistedrequest.write(response.content) + twistedrequest.finish() + + def _on_error(_failure): + content = _failure.getTraceback() if self.debug else 'Internal Error' + response = HttpResponse(content=str(_failure), status=500) + return _send_response(response) + + view, args, kwargs = self.urlresolver(self.urls, httprequest.path) + if not view: + response = HttpResponse(content='Not Found', status=404) + _send_response(response) + return server.NOT_DONE_YET + + deferred = mustbe_deferred(view, httprequest, *args, **kwargs) + deferred.addCallback(_send_response) + deferred.addErrback(_on_error) + if not deferred.timeoutCall: + deferred.setTimeout(self.timeout) + + return server.NOT_DONE_YET + + diff --git a/scrapy/trunk/scrapy/contrib/web/stats.py b/scrapy/trunk/scrapy/contrib/web/stats.py new file mode 100644 index 000000000..b86961482 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/web/stats.py @@ -0,0 +1,55 @@ +from functools import partial +from scrapy.core.exceptions import NotConfigured +from scrapy.utils.serialization import serialize +from scrapy.conf import settings + +from .site import WebSite, WebResource +from .json import JsonResponse + +STATS_WSPORT = settings.getint('STATS_WSPORT', 8089) +STATS_WSTIMEOUT = settings.getint('STATS_WSTIMEOUT', 15) + +jsonserialize = partial(serialize, format='json') + + +def stats(request): + service = request.ARGS.get('service', 'getstats') + if service == 'getstats': + domain = request.ARGS.get('domain') + count = request.ARGS.get('count', 1) + if not domain: + path = request.ARGS.get('path') + offset = request.ARGS.get('offset') + order = request.ARGS.get('order', 'domain') + olist = request.ARGS.get('olist', 'ASC') + + from scrapy.store.db import DomainDataHistory + ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') + + if service == 'countdomains': + stats = [ddh.domain_count()] + elif domain: + stats = list(ddh.get(domain, count=int(count))) + else: + stats = list(ddh.getlast_alldomains(count, offset, order, olist, path=path)) + + content = {'data': stats if stats else "No stats found"} + return JsonResponse(content=content, serialize=jsonserialize) + + +urlmapping = ( + ('^ws/tools/stats/$', stats), + ) + + +class StatsService(WebSite): + def __init__(self): + if not settings['WS_ENABLED']: + raise NotConfigured + + if not settings['SCRAPING_DB']: + print "SCRAPING_DB setting is required for the stats web service" + raise NotConfigured + + resource = WebResource(urlmapping, timeout=STATS_WSTIMEOUT) + WebSite.__init__(self, port=STATS_WSPORT, resource=resource) diff --git a/scrapy/trunk/scrapy/contrib/webconsole/__init__.py b/scrapy/trunk/scrapy/contrib/webconsole/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py b/scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py new file mode 100644 index 000000000..e90b9ab30 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/webconsole/enginestatus.py @@ -0,0 +1,27 @@ +""" +Scheduler information module for Scrapy webconsole +""" +from pydispatch import dispatcher +from scrapy.core.engine import scrapyengine +from scrapy.management.web import banner + +class EngineStatus(object): + webconsole_id = 'enginestatus' + webconsole_name = 'Engine status' + + def __init__(self): + from scrapy.management.web import webconsole_discover_module + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def webconsole_render(self, wc_request): + s = banner(self) + s += "
    \n"
+        s += scrapyengine.getstatus()
+        s += "
    \n" + s += "\n" + s += "\n" + + return s + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/webconsole/livestats.py b/scrapy/trunk/scrapy/contrib/webconsole/livestats.py new file mode 100644 index 000000000..b0e7fa037 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/webconsole/livestats.py @@ -0,0 +1,79 @@ +""" +Live statistics extension +""" +from datetime import datetime +from pydispatch import dispatcher +from scrapy.core import signals +from scrapy.core.engine import scrapyengine +from scrapy.management.web import banner + +class SpiderStats(object): + def __init__(self): + self.scraped = 0 + self.crawled = 0 + self.started = None + self.finished = None + +class LiveStats(object): + webconsole_id = 'livestats' + webconsole_name = 'Spider live statistics of current run' + + def __init__(self): + self.domains = {} + dispatcher.connect(self.domain_open, signal=signals.domain_open) + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + dispatcher.connect(self.item_scraped, signal=signals.item_scraped) + dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) + + from scrapy.management.web import webconsole_discover_module + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def domain_open(self, domain, spider): + pstats = SpiderStats() + self.domains[spider.domain_name] = pstats + pstats.started = datetime.now() + pstats.finished = None + + def domain_closed(self, domain, spider): + self.domains[spider.domain_name].finished = datetime.now() + + def item_scraped(self, item, spider): + self.domains[spider.domain_name].scraped += 1 + + def response_downloaded(self, response, spider): + self.domains[spider.domain_name].crawled += 1 + + def webconsole_render(self, wc_request): + sch = scrapyengine.scheduler + dwl = scrapyengine.downloader + + totdomains = totscraped = totcrawled = totscheduled = totactive = totpending = 0 + s = banner(self) + s += "\n" + s += "\n" + for d in sorted(self.domains.keys()): + scheduled = len(sch.pending_requests[d]) if d in sch.pending_requests else 0 + active, pending = len(dwl.active_requests(d)), len(dwl.request_queue(d)) + stats = self.domains[d] + runtime = stats.finished - stats.started if stats.finished else datetime.now() - stats.started + + s += '\n' % \ + (d, stats.scraped, stats.crawled, scheduled, pending, active, str(stats.started), str(stats.finished), str(runtime)) + + totdomains += 1 + totscraped += stats.scraped + totcrawled += stats.crawled + totscheduled += scheduled + totactive += active + totpending += pending + s += '\n' % \ + (totdomains, totscraped, totcrawled, totscheduled, totactive, totpending) + s += "
    DomainItems
    Scraped    		Pages
    Crawled    		Scheduler
    Pending    		Downloader
    Pending    		Downloader
    Active    		Start timeFinish timeRun time
    %s%d%d%d%d%d%s%s%s
    %d domains%d%d%d%d%d
    \n" + + s += "\n" + s += "\n" + + return s + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/webconsole/schedstats.py b/scrapy/trunk/scrapy/contrib/webconsole/schedstats.py new file mode 100644 index 000000000..36a207bf3 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/webconsole/schedstats.py @@ -0,0 +1,41 @@ +""" +Scheduler information module for Scrapy webconsole +""" +from pydispatch import dispatcher +from scrapy.core import signals +from scrapy.core.engine import scrapyengine +from scrapy.spider import spiders +from scrapy.management.web import banner + +class SchedulerStats(object): + webconsole_id = 'scheduler' + webconsole_name = 'Scheduler queue' + + def __init__(self): + from scrapy.management.web import webconsole_discover_module + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def webconsole_render(self, wc_request): + s = banner(self) + s += "
      \n" + for domain, requests in scrapyengine.scheduler.pending_requests.iteritems(): + s += "
    • \n" + s += "%s (%s pages)\n" % (domain, len(requests)) + s += "
        \n" + # requests is a tuple of request and deffered now, as I understand + for r, d in requests: + if hasattr(r, 'url'): + s += "
      • %s
        • \n" % (r.url, r.url) + #else: + # s += "
      • %s
        • \n" % (repr(r)) + s += "
      \n" + s += "
      • \n" + s += "
    \n" + + s += "\n" + s += "\n" + + return s + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py b/scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py new file mode 100644 index 000000000..ac93defac --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/webconsole/spiderctl.py @@ -0,0 +1,208 @@ +""" +Extensions for allowing spider control from web console +""" + +from pydispatch import dispatcher + +from scrapy.core import signals +from scrapy.core.manager import scrapymanager +from scrapy.core.engine import scrapyengine +from scrapy.spider import spiders +from scrapy.management.web import banner +from scrapy.conf import settings + +class Spiderctl(object): + webconsole_id = 'spiderctl' + webconsole_name = 'Spider control panel' + + def __init__(self): + self.running = set() + self.finished = set() + dispatcher.connect(self.domain_open, signal=signals.domain_open) + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + + from scrapy.management.web import webconsole_discover_module + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def domain_open(self, domain, spider): + self.running.add(domain) + + def domain_closed(self, domain, spider): + if domain in self.running: + self.running.remove(domain) + self.finished.add(domain) + + def webconsole_render(self, wc_request): + if wc_request.args: + changes = self.webconsole_control(wc_request) + + enabled_domains = spiders.asdict(include_disabled=False).keys() + self.scheduled = set(scrapyengine.scheduler.pending_domains_count) + self.not_scheduled = [d for d in enabled_domains if d not in self.scheduled + and d not in self.running + and d not in self.finished] + + s = banner(self) + s += '

    Reload spiders

    \n' + s += '\n' + s += "\n" % \ + (len(self.running), + settings['CONCURRENT_DOMAINS'], + len(self.scheduled), + len(self.finished), + len(self.not_scheduled)) + s += "\n" + + # running + s += "\n" + + # scheduled + s += "\n" + + # finished + s += "\n" + + # not scheduled + s += "\n" + + s += "\n" + s += "\n" + + s += "\n" + + s += "\n' + + s += "\n" + + s += "\n" + + s += "\n" + + s += "
    Running (%d/%d)Scheduled (%d)Finished (%d)Not scheduled (%d)
    \n" + s += '
    \n' + s += '
    \n' + s += '
    ' + s += '\n' + s += '
    \n' + s += "    		\n" + s += '
    \n' + s += '
    \n' + s += '
    ' + s += '\n' + s += '
    \n' + + s += "    		\n" + s += '
    \n' + s += '
    \n' + s += '
    ' + s += '\n' + s += '
    \n' + s += "    		\n" + s += '
    \n' + s += '
    \n' + s += '
    ' + s += '\n' + s += '
    \n' + s += "
     \n" + s += '
    \n' + s += "\n" + s += '
    \n' + s += '\n' + s += '
    \n' + s += '    		 \n" + s += '
    \n' + s += "\n" + s += '
    \n' + s += '\n' + s += '
    \n' + s += "
    \n" + + if wc_request.args: + s += changes + + s += "\n" + s += "\n" + + return s + + def webconsole_control(self, wc_request): + args = wc_request.args + s = "
    \n" + if "reloadspiders" in args: + scrapymanager.reload_spiders() + s += "

    Spiders reloaded

    \n" + return s + + if "stop_running_domains" in args: + s += "

    " + s += "Stopped spiders:

    • %s
    " % "
  • ".join(args["stop_running_domains"]) + for domain in args["stop_running_domains"]: + scrapyengine.close_domain(domain) + s += "

    " + if "remove_pending_domains" in args: + removed = [] + for domain in args["remove_pending_domains"]: + if scrapyengine.scheduler.remove_pending_domain(domain): + removed.append(domain) + if removed: + s += "

    " + s += "Removed scheduled spiders:

    • %s
    " % "
  • ".join(args["remove_pending_domains"]) + s += "

    " + if "bulk_remove_domains" in args: + scheduled = [] + to_remove = set([d.strip() for d in args["bulk_remove_domains"][0].split("\n")]) + removed = set([d for d in scrapyengine.scheduler.pending_domains if d in to_remove]) + scrapyengine.scheduler.pending_domains = [d for d in execengine.scheduler.pending_domains if d not in removed] + if removed: + s += "

    " + s += "Removed:

    • %s
    " % "
  • ".join(removed) + s += "

    " + s += "

    " + s += "Not removed:

    • %s
    " % "
  • ".join(to_remove - removed) + s += "

    " + if "add_pending_domains" in args: + for domain in args["add_pending_domains"]: + if domain not in scrapyengine.scheduler.pending_domains_count: + scrapymanager.crawl(domain) + s += "

    " + s += "Scheduled spiders:

    • %s
    " % "
  • ".join(args["add_pending_domains"]) + s += "

    " + if "bulk_schedule_domains" in args: + to_schedule = set([d.strip() for d in args["bulk_schedule_domains"][0].split("\n")]) + scheduled = set() + for domain in to_schedule: + if domain in enabled_domains and domain not in scrapyengine.scheduler.pending_domains_count: + scrapymanager.crawl(domain) + scheduled.add(domain) + if scheduled: + s += "

    " + s += "Scheduled:

    • %s
    " % "
  • ".join(scheduled) + s += "

    " + s += "

    " + s += "Not scheduled:

    • %s
    " % "
  • ".join(to_schedule - scheduled) + s += "

    " + if "rerun_finished_domains" in args: + for domain in args["rerun_finished_domains"]: + if domain not in scrapyengine.scheduler.pending_domains_count: + scrapymanager.crawl(domain) + self.finished.remove(domain) + s += "

    " + s += "Re-scheduled finished spiders:

    • %s
    " % "
  • ".join(args["rerun_finished_domains"]) + s += "

    " + + return s + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py b/scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py new file mode 100644 index 000000000..3cc451ec8 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/webconsole/spiderstats.py @@ -0,0 +1,80 @@ +""" +Extension for persistent statistics by domain +""" +import re +import pprint + +from pydispatch import dispatcher + +from scrapy.spider import spiders +from scrapy.core.exceptions import NotConfigured +from scrapy.management.web import banner +from scrapy.conf import settings + +class SpiderStats(object): + webconsole_id = 'spiderstats' + webconsole_name = 'Spider stats (all-time)' + + stats_mainpage = [ + ('item_scraped_count', "Items scraped
    (last run)"), + ('item_passed_count', "Items passed
    (last run)"), + ('response_count', "Pages crawled
    (last run)"), + ('start_time', "Start time
    (last run)"), + ('finish_time', "Finish time
    (last run)"), + ('finish_status', "Finish status
    (last run)"), + ] + + PATH_RE = re.compile("/spiderstats/([^/]+)") + + def __init__(self): + if not settings['SCRAPING_DB']: + raise NotConfigured("Requires SCRAPING_DB setting") + + from scrapy.store.db import DomainDataHistory + self.ddh = DomainDataHistory(settings['SCRAPING_DB'], 'domain_data_history') + + from scrapy.management.web import webconsole_discover_module + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def webconsole_render(self, wc_request): + m = self.PATH_RE.search(wc_request.path) + if m: + return self.render_domain(m.group(1)) + else: + return self.render_main() + + def render_main(self): + s = banner(self) + s += "\n" + s += "\n" + s += "" + for path, title in self.stats_mainpage: + s += "" % title + s += "\n" + for domain in spiders.asdict().keys(): + s += "\n" + s += "" % (domain, domain) + for path, title in self.stats_mainpage: + stat = self.ddh.getlast(domain, path=path) + value = stat[1] if stat else "None" + s += "" % value + s += "\n" + s += "
    Domain%s
    %s%s
    \n" + s += "\n" + s += "\n" + + return str(s) + + def render_domain(self, domain): + s = banner(self) + stats = self.ddh.getall(domain) + s += "
    \n"
+        s += pprint.pformat(list(stats))
+        s += "
    \n" + s += "\n" + s += "\n" + + return str(s) + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/contrib/webconsole/stats.py b/scrapy/trunk/scrapy/contrib/webconsole/stats.py new file mode 100644 index 000000000..f58cb50e8 --- /dev/null +++ b/scrapy/trunk/scrapy/contrib/webconsole/stats.py @@ -0,0 +1,26 @@ +import pprint + +from pydispatch import dispatcher + +from scrapy.stats import stats +from scrapy.management.web import banner, webconsole_discover_module + +class StatsDump(object): + webconsole_id = 'stats' + webconsole_name = 'StatsCollector dump' + + def __init__(self): + dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) + + def webconsole_render(self, wc_request): + s = banner(self) + s += "
    \n"
+        s += pprint.pformat(stats)
+        s += "
    \n" + s += "\n" + s += "\n" + + return str(s) + + def webconsole_discover_module(self): + return self diff --git a/scrapy/trunk/scrapy/core/__init__.py b/scrapy/trunk/scrapy/core/__init__.py new file mode 100644 index 000000000..59e07e643 --- /dev/null +++ b/scrapy/trunk/scrapy/core/__init__.py @@ -0,0 +1,3 @@ +""" +Scrapy core library classes and functions. +""" diff --git a/scrapy/trunk/scrapy/core/downloader/__init__.py b/scrapy/trunk/scrapy/core/downloader/__init__.py new file mode 100644 index 000000000..cecc64c8e --- /dev/null +++ b/scrapy/trunk/scrapy/core/downloader/__init__.py @@ -0,0 +1 @@ +from scrapy.core.downloader.manager import Downloader diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/trunk/scrapy/core/downloader/handlers.py new file mode 100644 index 000000000..1862f191a --- /dev/null +++ b/scrapy/trunk/scrapy/core/downloader/handlers.py @@ -0,0 +1,79 @@ +""" +Download handlers for different schemes +""" +from __future__ import with_statement + +import os +import urlparse + +from twisted.web.client import HTTPClientFactory +from twisted.internet import defer, reactor +from twisted.web import error as web_error + +from scrapy.core import signals +from scrapy.http import Request, Response, Headers +from scrapy.core.exceptions import UsageError, HttpException +from scrapy.utils.misc import defer_succeed +from scrapy.conf import settings + +def download_any(request, spider): + u = urlparse.urlparse(request.url) + if u.scheme == 'file': + return download_file(request, spider) + elif u.scheme in ('http', 'https'): + return download_http(request, spider) + else: + raise UsageError("Unsupported scheme '%s' in URL: <%s>" % (u.scheme, request.url)) + +def download_http(request, spider): + """This functions handles http/https downloads""" + url = urlparse.urldefrag(request.url)[0] + + agent = request.headers.get('user-agent', settings.get('USER_AGENT')) + request.headers.pop('user-agent', None) # remove user-agent if already present + factory = HTTPClientFactory(url=str(url), # never pass unicode urls to twisted + method=request.method, + postdata=request.body, + headers=request.headers, + agent=agent, + cookies=request.cookies, + timeout=getattr(spider, "download_timeout", None) or settings.getint('DOWNLOAD_TIMEOUT'), + followRedirect=False) + + def _response(body): + body = body or '' + status = factory.status + parent = request.headers.get('Referer') + headers = Headers(factory.response_headers) + r = Response(domain=spider.domain_name, url=request.url, headers=headers, status=status, body=body, parent=parent) + signals.send_catch_log(signal=signals.request_uploaded, sender='download_http', request=request, spider=spider) + signals.send_catch_log(signal=signals.response_downloaded, sender='download_http', response=r, spider=spider) + return r + + def _on_success(body): + return _response(body) + + def _on_error(_failure): + ex = _failure.value + if isinstance(ex, web_error.Error): # HttpException + raise HttpException(ex.status, ex.message, _response(ex.response)) + return _failure + + factory.noisy = False + factory.deferred.addCallbacks(_on_success, _on_error) + + u = urlparse.urlparse(request.url) + if u.scheme == 'https' : + from twisted.internet import ssl + contextFactory = ssl.ClientContextFactory() + reactor.connectSSL(u.hostname, u.port or 443, factory, contextFactory) + else: + reactor.connectTCP(u.hostname, u.port or 80, factory) + return factory.deferred + +def download_file(request, spider) : + """Return a deferred for a file download.""" + filepath = request.url.split("file://")[1] + with open(filepath) as f: + response = Response(domain=spider.domain_name, url=request.url, body=f.read()) + return defer_succeed(response) diff --git a/scrapy/trunk/scrapy/core/downloader/manager.py b/scrapy/trunk/scrapy/core/downloader/manager.py new file mode 100644 index 000000000..b3b367b7a --- /dev/null +++ b/scrapy/trunk/scrapy/core/downloader/manager.py @@ -0,0 +1,197 @@ +""" +Download web pages using asynchronous IO +""" + +import datetime + +from twisted.internet import reactor, defer + +from scrapy.core.exceptions import IgnoreRequest +from scrapy.spider import spiders +from scrapy.core.downloader.handlers import download_any +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.core import log +from scrapy.conf import settings +from scrapy.utils.misc import chain_deferred + + +class SiteDetails(object): + """This is a simple data record that encapsulates the details we hold on + each domain which we are scraping. + """ + def __init__(self, download_delay=None, max_concurrent_requests=2): + self.download_delay = download_delay + self.max_concurrent_requests = max_concurrent_requests if not download_delay else 1 + + self.queue = [] + self.active = set() + self.downloading = set() + self.closed = False + self.lastseen = None + + def is_idle(self): + return not (self.active or self.downloading) + + def capacity(self): + return self.max_concurrent_requests - len(self.downloading) + + +class Downloader(object): + """Maintain many concurrent downloads and provide an HTTP abstraction + We will have a limited number of connections per domain and scrape many domains in + parallel. + + request(..) should be called to request resources using http, https or file + protocols. + """ + + def __init__(self) : + self.sites = {} + self.middleware = DownloaderMiddlewareManager() + self.middleware.download_function = self.enqueue + self.download_function = download_any + + def fetch(self, request, spider): + """ Main method to use to request a download + + This method includes middleware mangling. Middleware can returns a + Response object, then request never reach downloader queue, and it will + not be downloaded from site. + """ + domain = spider.domain_name + site = self.sites[domain] + if not site or site.closed: + raise IgnoreRequest('Unable to fetch (domain already closed): %s' % request) + + site.active.add(request) + def _deactivate(_): + site.active.remove(request) + return _ + dwld = self.middleware.download(request, spider) + dwld.addBoth(_deactivate) + return dwld + + def enqueue(self, request, spider): + """ Enqueue a Request for a effective download from site + """ + domain = spider.domain_name + site = self.sites.get(domain) + if not site or site.closed: + raise IgnoreRequest('Trying to enqueue %s from closed site %s' % (request, domain)) + + deferred = defer.Deferred() + site.queue.append((request, deferred)) + self.process_queue(spider) + return deferred + + def process_queue(self, spider): + """ Effective download requests from site queue + """ + domain = spider.domain_name + site = self.sites.get(domain) + if not site: + return + + # download delay handling + now = datetime.datetime.now() + if site.download_delay and site.lastseen: + delta = now - site.lastseen + penalty = site.download_delay - delta.seconds + if penalty > 0: + reactor.callLater(penalty, self.process_queue, spider=spider) + return + site.lastseen = now + + while site.queue and site.capacity()>0: + request, deferred = site.queue.pop(0) + self._download(request, spider, deferred) + + if site.closed and site.is_idle(): + # XXX: Remove scrapyengine reference + del self.sites[domain] + from scrapy.core.engine import scrapyengine + scrapyengine.closed_domain(domain=domain) + + def _download(self, request, spider, deferred): + log.msg('Activating %s' % request.traceinfo(), log.TRACE) + domain = spider.domain_name + site = self.sites.get(domain) + site.downloading.add(request) + + def _remove(result): + log.msg('Deactivating %s' % request.traceinfo(), log.TRACE) + site.downloading.remove(request) + return result + + def _finish(result): + self.process_queue(spider) + + from scrapy.utils.misc import mustbe_deferred + dwld = mustbe_deferred(self.download_function, request, spider) + dwld.addBoth(_remove) + chain_deferred(dwld, deferred) + dwld.addBoth(_finish) + + def open_domain(self, domain): + """Allocate resources to begin processing a domain""" + spider = spiders.fromdomain(domain) + if domain in self.sites: # reopen + self.sites[domain].closed = False + return + + # Instanciate site specific handling based on info provided by spider + delay = getattr(spider, 'download_delay', None) + maxcr = getattr(spider, 'max_concurrent_requests', settings.getint('REQUESTS_PER_DOMAIN')) + site = SiteDetails(download_delay=delay, max_concurrent_requests=maxcr) + self.sites[domain] = site + + def close_domain(self, domain): + """Free any resources associated with the given domain""" + log.msg("Downloader closing domain %s" % domain, log.TRACE, domain=domain) + site = self.sites.get(domain) + if site: + site.closed = True + spider = spiders.fromdomain(domain) + self.process_queue(spider) + else: + log.msg('Domain %s already closed' % domain, log.TRACE, domain=domain) + + # Most of the following functions must be reviewed to decide if are really needed + def domain_is_open(self, domain): + return domain in self.sites + + def lastseen(self, domain): + if domain in self.sites: + return self.sites[domain].lastseen + + def outstanding(self, domain): + """The number of outstanding requests for a domain + This includes both active requests and pending requests. + """ + site = self.sites.get(domain) + if site: + return len(site.active) + len(site.queue) + + def domain_is_idle(self, domain): + return not self.outstanding(domain) + + def request_queue(self, domain): + site = self.sites.get(domain) + return site.queue if site else [] + + def active_requests(self, domain): + site = self.sites.get(domain) + return site.active if site else [] + + def has_capacity(self): + """Does the downloader have capacity to handle more domains""" + return len(self.sites) < settings.getint('CONCURRENT_DOMAINS') + + def is_idle(self): + return not self.sites + + # deprecated + def clear_requests(self, domain): + log.msg("Downloader clearing request for domain %s" % domain, log.TRACE, domain=domain) + self.sites[domain].queue = [] + diff --git a/scrapy/trunk/scrapy/core/downloader/middleware.py b/scrapy/trunk/scrapy/core/downloader/middleware.py new file mode 100644 index 000000000..3f952f27b --- /dev/null +++ b/scrapy/trunk/scrapy/core/downloader/middleware.py @@ -0,0 +1,169 @@ +""" +request-response middleware extension +""" +from scrapy.core import log +from scrapy.http import Request, Response +from scrapy.core.exceptions import NotConfigured +from scrapy.utils.misc import load_class, mustbe_deferred +from scrapy.core.downloader.handlers import download_any +from scrapy.conf import settings + +class DownloaderMiddlewareManager(object): + """Request-Response Middleware Manager + + Middleware is a framework of hooks into Scrapy's request/response + processing. It's a light, low-level "spider" system for globally altering + Scrapy's input and/or output. + + Middleware is heavily based on Django middleware system, at the point that + it tries to mimic Django middleware behaviour. For Scrapy, the Django's + view function has the same meaning of the final download handler function + to use for the request's url. + + To activate a middleware component, add it to the DOWNLOADER_MIDDLEWARES list + in your Scrapy settings. In DOWNLOADER_MIDDLEWARES, each middleware component + is represented by a string: the full Python path to the middleware's class + name. For example: + + DOWNLOADER_MIDDLEWARES = ( + 'scrapy.contrib.middleware.common.SpiderMiddleware', + 'scrapy.contrib.middleware.common.CommonMiddleware', + 'scrapy.contrib.middleware.redirect.RedirectMiddleware', + 'scrapy.contrib.middleware.cache.CacheMiddleware', + ) + + Writing your own middleware is easy. Each middleware component is a single + Python class that defines one or more of the following methods: + + + process_request(self, request, spider) + + `request` is a Request object. + `spider` is a BaseSpider object + + This method is called in each request until scrapy decides which + download function to use. + + process_request() should return either None, Response or Request. + + If returns None, Scrapy will continue processing this request, + executing any other middleware and, then, the appropiate download + function. + + If returns a Response object, Scrapy won't bother calling ANY other + request or exception middleware, or the appropriate download function; + it'll return that Response. Response middleware is always called on + every response. + + If returns a Request object, returned request is used to instruct a + redirection. Redirection is handled inside middleware scope, and + original request don't finish until redirected request is completed. + + + process_response(self, request, response, spider): + + `request` is a Request object + `response` is a Response object + `spider` is a BaseSpider object + + process_response MUST return a Response object. It could alter the given + response, or it could create a brand-new Response. + + + process_exception(self, request, exception, spider) + + `request` is a Request object. + `exception` is an Exception object + `spider` is a BaseSpider object + + Scrapy calls process_exception() when a download handler or + process_request middleware raises an exception. + + process_exception() should return either None, Response or Request object. + + if it returns None, Scrapy will continue processing this exception, + executing any other exception middleware, until no middleware left and + default exception handling kicks in. + + If it returns a Response object, the response middleware kicks in, and + won't bother calling ANY other exception middleware. + + If it returns a Request object, returned request is used to instruct a + immediate redirection. Redirection is handled inside middleware scope, + and original request don't finish until redirected request is + completed. This stop process_exception middleware as returning + Response does. + + """ + def __init__(self): + self.loaded = False + self.request_middleware = [] + self.response_middleware = [] + self.exception_middleware = [] + self.load() + self.download_function = download_any + + def _add_middleware(self, mw): + if hasattr(mw, 'process_request'): + self.request_middleware.append(mw.process_request) + if hasattr(mw, 'process_response'): + self.response_middleware.insert(0, mw.process_response) + if hasattr(mw, 'process_exception'): + self.exception_middleware.insert(0, mw.process_exception) + + def load(self): + """Load middleware defined in settings module + """ + mws = [] + for mwpath in settings.getlist('DOWNLOADER_MIDDLEWARES') or (): + cls = load_class(mwpath) + if cls: + try: + mw = cls() + self._add_middleware(mw) + mws.append(mw) + except NotConfigured: + pass + log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws])) + self.loaded = True + + def download(self, request, spider): + def process_request(request): + for method in self.request_middleware: + response = method(request=request, spider=spider) + assert response is None or isinstance(response, (Response, Request)), \ + 'Middleware %s.process_request must returns None, Response or Request, got %s ' % \ + (method.im_self.__class__.__name__, response.__class__.__name__) + if response: + return response + return self.download_function(request=request, spider=spider) + + def process_response(response): + assert response is not None, 'Received None in process_response' + if isinstance(response, Request): + return response + + for method in self.response_middleware: + response = method(request=request, response=response, spider=spider) + assert isinstance(response, (Response, Request)), \ + 'Middleware %s.process_response must returns Response or Request, got %s ' % \ + (method.im_self.__class__.__name__, type(response)) + if isinstance(response, Request): + return response + return response + + def process_exception(_failure): + exception = _failure.value + for method in self.exception_middleware: + response = method(request=request, exception=exception, spider=spider) + assert response is None or isinstance(response, (Response, Request)), \ + 'Middleware %s.process_exception must returns None, Response or Request, got %s ' % \ + (method.im_self.__class__.__name__, type(response)) + if response: + return response + return _failure + + deferred = mustbe_deferred(process_request, request) + deferred.addErrback(process_exception) + deferred.addCallback(process_response) + return deferred diff --git a/scrapy/trunk/scrapy/core/engine.py b/scrapy/trunk/scrapy/core/engine.py new file mode 100644 index 000000000..0b9bd26ed --- /dev/null +++ b/scrapy/trunk/scrapy/core/engine.py @@ -0,0 +1,541 @@ +""" +Core execution engine for the web crawling framework. This controls the +scheduler, downloader and spiders. +""" +from datetime import datetime + +from twisted.internet import defer, reactor, task +from twisted.python.failure import Failure +from pydispatch import dispatcher + +from scrapy.core import signals, log +from scrapy.core.scheduler import Scheduler +from scrapy.core.downloader import Downloader +from scrapy.http import Response, Request +from scrapy.core.exceptions import IgnoreRequest, HttpException, DontCloseDomain +from scrapy.item import ScrapedItem +from scrapy.item.pipeline import ItemPipeline +from scrapy.spider import spiders +from scrapy.spider.middleware import SpiderMiddlewareManager +from scrapy.utils.misc import chain_deferred, defer_succeed, mustbe_deferred +from scrapy.conf import settings + + +class ExecutionEngine(object): + """ + The Execution Engine controls execution of the scraping process. + + The process begins with the _mainloop() method, which is called + periodically to add more domains to scrape. It adds the first page for a + domain to the scheduler by calling _schedule_page and calls + process_scheduled_requests, which starts the scraping process for a domain. + + The process_scheduled_requests method asks the scheduler for the next + available page for a given domain and requests it from the downloader. The + downloader will execute a callback function that was added in the + _schedule_page method. This callback with process the output from the + spider and then call process_scheduled_requests to continue the scraping + process for that domain. + """ + + # back off downloading more pages if we exceed this backlog size + DOWNLOADER_BACKLOG = 10 + + def __init__(self): + self.configured = False + self.keep_alive = False + self.domain_close_delay = settings.getint('DOMAIN_CLOSE_DELAY') # seconds after an idle domain will be closed + self.initializing = set() # domais in intialization state + self.cancelled = set() # domains in cancelation state + self.debug_mode = settings.getbool('ENGINE_DEBUG') + self.tasks = [] + self.ports = [] + self.running = False + self.paused = False + + def configure(self, scheduler=None, downloader=None): + """ + Configure execution engine with the given scheduling policy and downloader. + """ + self.scheduler = scheduler or Scheduler() + self.downloader = downloader or Downloader() + self.spidermiddleware = SpiderMiddlewareManager() + self._scraping = {} + self.pipeline = ItemPipeline() + # key dictionary of per domain lists of initial requests to scrape + self.starters = {} + + self.configured = True + + def addtask(self, function, interval, args=None, kwargs=None, now=False): + """ + Adds a looping task. Use this instead of twisted task.LooopingCall to + make sure the reactor is left in a clean state after the engine is + stopped. + """ + if not args: + args = [] + if not kwargs: + kwargs = {} + tsk = task.LoopingCall(function, *args, **kwargs) + self.tasks.append((tsk, interval, now)) + if self.running: + tsk.start(interval, now) + return tsk + + def removetask(self, tsk): + """Remove a looping task previously added with addtask() method""" + self.tasks = [(t, i, n) for (t, i, n) in self.tasks if t is not tsk] + tsk.stop() + + def listenTCP(self, *args, **kwargs): + if self.running: + self.ports.append(reactor.listenTCP(*args, **kwargs)) + else: + self.ports.append((args, kwargs)) + + def clean_reactor(self): + """Leaves the reactor in a clean state by removing all pending tasks + and listening ports. It can only be called when the engine is not + running. + """ + if not self.running: + for tsk, _, _ in self.tasks: + if tsk.running: + tsk.stop() + self.tasks = [] + for p in [p for p in self.ports if not isinstance(p, tuple)]: + p.stopListening() + self.ports = [] + + def start(self): + """Start the execution engine""" + if not self.running: + reactor.callWhenRunning(self._mainloop) + self.start_time = datetime.now() + signals.send_catch_log(signal=signals.engine_started, sender=self.__class__) + self.addtask(self._mainloop, 5.0) + for tsk, interval, now in self.tasks: + tsk.start(interval, now) + for args, kwargs in [t for t in self.ports if isinstance(t, tuple)]: + reactor.listenTCP(*args, **kwargs) + self.running = True + reactor.run() # blocking call + + def stop(self): + """Stop the execution engine""" + if self.running: + self.running = False + for domain in self.open_domains: + spider = spiders.fromdomain(domain) + signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, domain=domain, spider=spider, status='cancelled') + for tsk, _, _ in self.tasks: # stop looping calls + tsk.stop() + self.tasks = [] + for p in [p for p in self.ports if not isinstance(p, tuple)]: + p.stopListening() + reactor.stop() + signals.send_catch_log(signal=signals.engine_stopped, sender=self.__class__) + + def pause(self): + """Pause the execution engine""" + self.paused = True + + def resume(self): + """Resume the execution enine""" + self.paused = False + + def is_idle(self): + return self.scheduler.is_idle() and self.pipeline.is_idle() and self.downloader.is_idle() and not self._scraping + + def next_domain(self): + domain = self.scheduler.next_domain() + if domain: + spider = spiders.fromdomain(domain) + self.open_domain(domain, spider) + return domain + + def next_request(self, spider, breakloop=True): + """Scrape the next request for the domain passed. + + The next request to be scraped is retrieved from the scheduler and + requested from the downloader. + + The domain is closed if there are no more pages to scrape. + """ + if self.paused: + return reactor.callLater(5, self.next_request, spider) + + if breakloop: + # delaying make reentrant call to next_request safe + return reactor.callLater(0, self.next_request, spider, breakloop=False) + + domain = spider.domain_name + + # check that the engine is still running and domain is open + if not self.running: + return + + # outstanding requests in downloader queue + outstanding = self.downloader.outstanding(domain) + if outstanding > self.DOWNLOADER_BACKLOG: + return + + # Next pending request from scheduler + request, deferred = self.scheduler.next_request(domain) + if request: + try: + dwld = self.download(request, spider) + except IgnoreRequest, ex: + log.msg(ex.message, log.WARNING, domain=domain) + except Exception, ex: + log.exc("Bug in download code: %s" % request, domain=domain) + self._domain_idle(domain) + else: + chain_deferred(dwld, deferred) + else: + if self.domain_is_idle(domain): + self._domain_idle(domain) + + def domain_is_idle(self, domain): + scraping = self._scraping.get(domain) + pending = self.scheduler.domain_has_pending(domain) + downloading = not self.downloader.domain_is_idle(domain) + haspipe = not self.pipeline.domain_is_idle(domain) + oninit = domain in self.initializing + if pending or downloading or haspipe or oninit or scraping: + return False + elif self.domain_close_delay: + lastseen = self.downloader.lastseen(domain) + if not lastseen: # domain not yet started + return False + now = datetime.now() + delta = now - lastseen + return (delta.seconds >= self.domain_close_delay) + else: + return True + + def domain_is_open(self, domain): + return domain in self.downloader.sites + + @property + def open_domains(self): + return self.downloader.sites.keys() + + def crawl(self, request, spider, priority=1, domain_priority=1): + domain = spider.domain_name + + def _process(response): + assert isinstance(response, (Response, Exception)) + + def _onpipelinefinish(pipe_result, item): + # _ can only be an item or a DropItem failure, since other + # failures are caught in ItemPipeine (item/pipeline.py) + if isinstance(pipe_result, Failure): + signals.send_catch_log(signal=signals.item_dropped, sender=self.__class__, item=item, spider=spider, response=response, exception=pipe_result.value) + else: + signals.send_catch_log(signal=signals.item_passed, sender=self.__class__, item=item, spider=spider, response=response, pipe_output=pipe_result) + self.next_request(spider) + + def _onsuccess(result): + for item in result or (): + if isinstance(item, ScrapedItem): + log.msg("Scraped %s in <%s>" % (item, request.url), log.DEBUG, domain=domain) + signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, item=item, spider=spider, response=response) + piped = self.pipeline.pipe(item, spider, response) # TODO: remove response + piped.addBoth(_onpipelinefinish, item) + elif isinstance(item, Request): + signals.send_catch_log(signal=signals.request_received, sender=self.__class__, request=item, spider=spider, response=response) + self.crawl(request=item, spider=spider, priority=priority) + else: + log.msg('Garbage found in spider output while processing %s, got type %s' % (request, type(item)), log.TRACE, domain=domain) + + def _onerror(_failure): + if not isinstance(_failure.value, IgnoreRequest): + referer = None if not isinstance(response, Response) else response.request.headers.get('Referer', None) + log.msg("Error while spider was processing <%s> from <%s>: %s" % (request.url, referer, _failure), log.ERROR, domain=domain) + + def _bugtrap(_failure): + log.msg('FRAMEWORK BUG processing %s: %s' % (request, _failure), log.ERROR, domain=domain) + + scd = self.scrape(request, response, spider) + scd.addCallbacks(_onsuccess, _onerror) + scd.addErrback(_bugtrap) + scd.addBoth(lambda _:response) + return scd + + def _cleanfailure(_failure): + ex = _failure.value + if not isinstance(ex, IgnoreRequest): + log.msg("Unknown error propagated in %s: %s" % (request, _failure), log.ERROR, domain=domain) + request.deferred.addErrback(lambda _:None) + request.deferred.errback(_failure) #TODO: merge into spider middleware. + + schd = self.schedule(request, spider, priority, domain_priority) + schd.addCallbacks(_process, _cleanfailure) + return schd + + def scrape(self, request, response, spider): + domain = spider.domain_name + scd = self.spidermiddleware.scrape(request, response, spider) + self._scraping[domain].add(response) + def _remove(_): + self._scraping[domain].remove(response) + return _ + return scd.addBoth(_remove) + + def schedule(self, request, spider, priority=1, domain_priority=1): + domain = spider.domain_name + if not self.scheduler.domain_is_open(domain): + if self.debug_mode: + log.msg('Scheduling %s (delayed)' % request.traceinfo(), log.TRACE) + return self._add_starter(request, spider, domain_priority) + if self.debug_mode: + log.msg('Scheduling %s (now)' % request.traceinfo(), log.TRACE) + schd = self.scheduler.enqueue_request(domain, request, priority) + self.next_request(spider) + return schd + + def _mainloop(self): + """Add more domains to be scraped if the downloader has the capacity. + + If there is nothing else scheduled then stop the execution engine. + """ + if not self.running or self.paused: + return + + # main domain starter loop + while self.running and self.downloader.has_capacity(): + if not self.next_domain(): + return self._stop_if_idle() + + # purge idle domains - (domain_close_delay support) + if self.domain_close_delay: + for domain in self.downloader.sites: + if self.domain_is_idle(domain): + self._domain_idle(domain) + + def _add_starter(self, request, spider, domain_priority): + domain = spider.domain_name + if not self.scheduler.is_pending(domain): + self.scheduler.add_domain(domain, priority=domain_priority) + self.starters[domain] = [] + deferred = defer.Deferred() + self.starters[domain] += [(request, deferred)] + return deferred + + def _run_starters(self, spider): + domain = spider.domain_name + starters = self.starters.get(domain, []) + while starters: + request, deferred = starters.pop(0) + schd = self.schedule(request, spider) + chain_deferred(schd, deferred) + del self.starters[domain] + + def download(self, request, spider): + log.msg('Downloading %s' % request.traceinfo(), log.TRACE) + domain = spider.domain_name + + def _on_success(response): + """handle the result of a page download""" + assert isinstance(response, (Response, Request)) + log.msg("Requested %s" % request.traceinfo(), level=log.TRACE, domain=domain) + if isinstance(response, Response): + response.request = request # tie request to obtained response + log.msg("Crawled <%s> from <%s>" % (response.url, response.parent), level=log.DEBUG, domain=domain) + return response + elif isinstance(response, Request): + redirected = response # proper alias + schd = self.schedule(redirected, spider, priority=4) + chain_deferred(schd, redirected.deferred) + return schd + + def _on_error(_failure): + """handle an error processing a page""" + ex = _failure.value + if isinstance(ex, IgnoreRequest): + log.msg(_failure.getErrorMessage(), level=log.DEBUG, domain=domain) + return _failure + referer = request.headers.get('Referer', None) + errmsg = str(ex) if isinstance(ex, HttpException) else str(_failure) + log.msg("Downloading <%s> from <%s>: %s" % (request.url, referer, errmsg), log.ERROR, domain=domain) + return Failure(IgnoreRequest(str(ex))) + + def _on_complete(_): + self.next_request(spider) + + dwld = self.downloader.fetch(request, spider) + dwld.addCallbacks(_on_success, _on_error) + deferred = defer.Deferred() + chain_deferred(dwld, deferred) + dwld.addBoth(_on_complete) + return deferred + + def initialize(self, spider): + domain = spider.domain_name + if not hasattr(spider, 'init_domain'): + return defer_succeed(True) + + def _initialize(req): + if isinstance(req, Request): + _response = None + def _referer(response): + req.deferred.addCallback(_setreferer, response) + return response + + def _setreferer(result, response): + if isinstance(result, Request): + result.headers.setdefault('Referer', response.url) + return result + + def _onerror(_failure): + ex = _failure.value + if isinstance(ex, IgnoreRequest): + log.msg(ex.message, log.DEBUG, domain=domain) + else: + return _failure + + schd = self.schedule(req, spider) + schd.addCallback(_referer) + chain_deferred(schd, req.deferred) + schd.addErrback(_onerror) + schd.addBoth(_initialize) + return schd + return req + + def _bugtrap(_failure): + log.msg("Bug in %s init_domain code: %s" % (domain, _failure), log.ERROR, domain=domain) + + def _state(state): + self.initializing.remove(domain) + if state is True: + log.msg('Succeded initialization for %s' % domain, log.INFO, domain=domain) + else: + log.msg('Failed initialization for %s' % domain, log.INFO, domain=domain) + return state + + log.msg('Started initialization for %s' % domain, log.INFO, domain=domain) + self.initializing.add(domain) + req = spider.init_domain() + deferred = mustbe_deferred(_initialize, req) + deferred.addErrback(_bugtrap) + deferred.addCallback(_state) + return deferred + + def open_domain(self, domain, spider=None): + log.msg("Domain opened", domain=domain) + spider = spider or spiders.fromdomain(domain) + + self.scheduler.open_domain(domain) + self.downloader.open_domain(domain) + self.pipeline.open_domain(domain) + self._scraping[domain] = set() + signals.send_catch_log(signals.domain_open, sender=self.__class__, domain=domain, spider=spider) + + # init_domain + dfd = self.initialize(spider) + def _state(state): + if state is True: + signals.send_catch_log(signals.domain_opened, sender=self.__class__, domain=domain, spider=spider) + self._run_starters(spider) + else: + self._domain_idle(domain) + dfd.addCallback(_state) + + def close_domain(self, domain): + """Close (cancel) domain and clear all its outstanding requests""" + if domain not in self.cancelled: + self.cancelled.add(domain) + self._close_domain(domain) + + def _close_domain(self, domain): + self.downloader.close_domain(domain) + + def _domain_idle(self, domain): + """ + Called when a domain gets idle. This function is called when there are no + remaining pages to download or scheduled. It can be called multiple + times. If the some extensions raises a DontCloseDomain exception the + domain won't be closed and this function is garanteed to be called + again (at least once) for this domain. + """ + # we get a callback from the downloader which completes closing + #log.msg("Finishing scraping %s" % domain, domain=domain) + #from traceback import print_stack; print_stack() + + spider = spiders.fromdomain(domain) + try: + dispatcher.send(signal=signals.domain_idle, sender=self.__class__, domain=domain, spider=spider) + except DontCloseDomain: + return + except: + log.exc("Exception catched on domain_idle signal dispatch") + + if self.domain_is_idle(domain): + self._close_domain(domain) + + def _stop_if_idle(self): + """Call the stop method if the system has no outstanding tasks. """ + if self.is_idle() and not self.keep_alive: + self.stop() + + def closed_domain(self, domain): + """ + This function is called after the domain has been closed, and throws + the domain_closed signal which is meant to be used for cleaning up + purposes. In contrast to domain_idle, this function is called only + ONCE for each domain run. + """ + spider = spiders.fromdomain(domain) + self.scheduler.close_domain(domain) + self.pipeline.close_domain(domain) + del self._scraping[domain] + status = 'cancelled' if domain in self.cancelled else 'finished' + signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, domain=domain, spider=spider, status=status) + log.msg("Domain closed (%s)" % status, domain=domain) + self.cancelled.discard(domain) + self._mainloop() + + def getstatus(self): + """ + Return a report of the current engine status + """ + s = "Execution engine status\n\n" + + global_tests = [ + "datetime.now()-self.start_time", + "self.is_idle()", + "self.scheduler.is_idle()", + "len(self.scheduler.pending_domains_count)", + "self.downloader.is_idle()", + "len(self.downloader.sites)", + "self.downloader.has_capacity()", + "self.pipeline.is_idle()", + "len(self.pipeline.domaininfo)", + ] + domain_tests = [ + "self.domain_is_idle(domain)", + "self.scheduler.domain_has_pending(domain)", + "len(self.scheduler.pending_requests[domain])", + "self.downloader.outstanding(domain)", + "len(self.downloader.request_queue(domain))", + "len(self.downloader.active_requests(domain))", + "self.pipeline.domain_is_idle(domain)", + "len(self.pipeline.domaininfo[domain])", + ] + + for test in global_tests: + s += "%-47s : %s\n" % (test, eval(test)) + s += "\n" + for domain in self.downloader.sites: + s += "%s\n" % domain + for test in domain_tests: + s += " %-45s : %s\n" % (test, eval(test)) + + return s + + def st(self): # shortcut for printing engine status (useful in telnet console) + print self.getstatus() + +scrapyengine = ExecutionEngine() diff --git a/scrapy/trunk/scrapy/core/exceptions.py b/scrapy/trunk/scrapy/core/exceptions.py new file mode 100644 index 000000000..416559226 --- /dev/null +++ b/scrapy/trunk/scrapy/core/exceptions.py @@ -0,0 +1,44 @@ +""" +Scrapy core exceptions +""" + +# Internal + +class UsageError(Exception): + """Incorrect usage of the core API""" + pass + +class NotConfigured(Exception): + """Indicates a missing configuration situation""" + pass + +# HTTP and crawling + +class IgnoreRequest(Exception): + """Indicates a decision was made not to process a request""" + pass + +class DontCloseDomain(Exception): + """Request the domain not to be closed yet""" + pass + +class HttpException(Exception): + def __init__(self, status, message, response): + if not message: + from twisted.web import http + message = http.responses.get(int(status)) + + self.status = status + self.message = message + self.response = response + Exception.__init__(self, status, message, response) + + def __str__(self): + return '%s %s' % (self.status, self.message) + +# Items + +class DropItem(Exception): + """Drop item from the item pipeline""" + pass + diff --git a/scrapy/trunk/scrapy/core/log.py b/scrapy/trunk/scrapy/core/log.py new file mode 100644 index 000000000..801aaf7ab --- /dev/null +++ b/scrapy/trunk/scrapy/core/log.py @@ -0,0 +1,55 @@ +""" +Crawler logging functionality +""" +from twisted.python import log +import sys +from scrapy.conf import settings + +# Logging levels +levels = { + 0: "SILENT", + 1: "CRITICAL", + 2: "ERROR", + 3: "WARNING", + 4: "INFO", + 5: "DEBUG", + 6: "TRACE", +} + +BOT_NAME = settings['BOT_NAME'] + +# Set logging level module attributes +for v, k in levels.items(): + setattr(sys.modules[__name__], k, v) + +# default logging level +log_level = DEBUG + +started = False + +def start(logfile=None, loglevel=None, log_stdout=None): + """ Init logging """ + if started or not settings.getbool('LOG_ENABLED'): + return + + logfile = logfile or settings['LOGFILE'] + loglevel = loglevel or settings['LOGLEVEL'] + log_stdout = log_stdout or settings.getbool('LOG_STDOUT') + + file = open(logfile, 'w') if logfile else sys.stderr + level = int(getattr(sys.modules[__name__], loglevel)) if loglevel else DEBUG + log.startLogging(file, setStdout=log_stdout) + setattr(sys.modules[__name__], 'log_level', level) + setattr(sys.modules[__name__], 'started', True) + +def msg(message, level=INFO, component=BOT_NAME, domain=None): + """ Log message according to the level """ + component = "%s/%s" % (BOT_NAME, domain) if domain else component + if level <= log_level: + log.msg("%s: %s" % (levels[level], message), system=component) + +def exc(message, level=ERROR, component=BOT_NAME, domain=None): + from traceback import format_exc + message = message + '\n' + format_exc() + msg(message, level, component, domain) + diff --git a/scrapy/trunk/scrapy/core/mail.py b/scrapy/trunk/scrapy/core/mail.py new file mode 100644 index 000000000..0e2cb8551 --- /dev/null +++ b/scrapy/trunk/scrapy/core/mail.py @@ -0,0 +1,54 @@ +import smtplib + +from email.MIMEMultipart import MIMEMultipart +from email.MIMEBase import MIMEBase +from email.MIMEText import MIMEText +from email.Utils import COMMASPACE, formatdate +from email import Encoders + +from scrapy.core import log +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings + +class MailSender(object): + + def __init__(self, smtphost=None, mailfrom=None): + self.smtphost = smtphost if smtphost else settings['MAIL_HOST'] + self.mailfrom = mailfrom if mailfrom else settings['MAIL_FROM'] + + if not self.smtphost or not self.mailfrom: + raise NotConfigured("MAIL_HOST and MAIL_FROM settings are required") + + def send(self, to, subject, body, cc=None, attachs=None): + """ + Send mail to the given recipients + + - to: must be a list of email recipients + - attachs must be a list of tuples: (attach_name, mimetype, file_object) + - body and subjet must be a string + """ + + msg = MIMEMultipart() + msg['From'] = self.mailfrom + msg['To'] = COMMASPACE.join(to) + msg['Date'] = formatdate(localtime=True) + msg['Subject'] = subject + rcpts = to[:] + if cc: + rcpts.extend(cc) + msg['Cc'] = COMMASPACE.join(cc) + + msg.attach(MIMEText(body)) + + for attach_name, mimetype, f in (attachs or []): + part = MIMEBase(*mimetype.split('/')) + part.set_payload(f.read()) + Encoders.encode_base64(part) + part.add_header('Content-Disposition', 'attachment; filename="%s"' % attach_name) + msg.attach(part) + + smtp = smtplib.SMTP(self.smtphost) + smtp.sendmail(self.mailfrom, rcpts, msg.as_string()) + log.msg('Mail sent: To=%s Cc=%s Subject="%s"' % (to, cc, subject)) + smtp.close() + diff --git a/scrapy/trunk/scrapy/core/manager.py b/scrapy/trunk/scrapy/core/manager.py new file mode 100644 index 000000000..191303ce4 --- /dev/null +++ b/scrapy/trunk/scrapy/core/manager.py @@ -0,0 +1,145 @@ +import signal + +from scrapy.extension import extensions +from scrapy.core import log +from scrapy.http import Request +from scrapy.core.engine import scrapyengine +from scrapy.spider import spiders +from scrapy.utils.misc import load_class +from scrapy.utils.url import is_url +from scrapy.conf import settings + +class ExecutionManager(object): + """Process a list of sites or urls. + + This class should be used in a main for process a list of sites/urls. + + It extracts products and could be used to store results in a database or + just for testing spiders. + """ + def __init__(self): + self.interrupted = False + self.configured = False + + def configure(self, *args, **opts): + self._install_signals() + + extensions.load() + log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys())) + + scheduler = load_class(settings['SCHEDULER'])() + + scrapyengine.configure(scheduler=scheduler) + + self.prioritizer_class = load_class(settings['PRIORITIZER']) + + requests = self._parse_args(args) + self.priorities = self.prioritizer_class(requests.keys()) + + def crawl(self, *args): + """Schedule the given args for crawling. args is a list of urls or domains""" + + requests = self._parse_args(args) + # schedule initial requets to be scraped at engine start + for domain in requests or (): + spider = spiders.fromdomain(domain) + priority = self.priorities.get_priority(domain) + for request in requests[domain]: + scrapyengine.crawl(request, spider, domain_priority=priority) + + def runonce(self, *args, **opts): + """Run the engine until it finishes scraping all domains and then exit""" + self.configure(*args, **opts) + self.crawl(*args) + scrapyengine.start() + + def start(self, **opts): + """Start the scrapy server, without scheduling any domains""" + self.configure(**opts) + scrapyengine.keep_alive = True + scrapyengine.start() + + def stop(self): + """Stop the scrapy server, shutting down the execution engine""" + self.interrupted = True + scrapyengine.stop() + log.log_level = -999 # disable logging + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + if hasattr(signal, "SIGBREAK"): + signal.signal(signal.SIGBREAK, signal.SIG_IGN) + + def reload_spiders(self): + """ + Reload all enabled spiders except for the ones that are currently + running. + + """ + spiders.reload(skip_domains=scrapyengine.open_domains) + # reload priorities for the new domains + self.priorities = self.prioritizer_class(spiders.asdict(include_disabled=False).keys()) + + def _install_signals(self): + def sig_handler_terminate(signalinfo, param): + log.msg('Received shutdown request, waiting for deferreds to finish...', log.INFO) + self.stop() + + signal.signal(signal.SIGTERM, sig_handler_terminate) + # only handle SIGINT if there isn't already a handler (e.g. for Pdb) + if signal.getsignal(signal.SIGINT) == signal.default_int_handler: + signal.signal(signal.SIGINT, sig_handler_terminate) + # Catch Ctrl-Break in windows + if hasattr(signal, "SIGBREAK"): + signal.signal(signal.SIGBREAK, sig_handler_terminate) + + def _parse_args(self, args): + """ Parse crawl arguments and return a tuple of (domains, urls) """ + if not args: + args = [p.domain_name for p in spiders.enabled] + + requests, urls, sites = set(), set(), set() + for a in args: + if isinstance(a, Request): + requests.add(a) + elif is_url(a): + urls.add(a) + else: + sites.add(a) + + perdomain = {} + def _add(domain, request): + if domain not in perdomain: + perdomain[domain] = [] + perdomain[domain] += [request] + + # sites + for domain in sites: + spider = spiders.fromdomain(domain) + if not spider: + log.msg('Could not found spider for %s' % domain, log.ERROR) + continue + for url in self._start_urls(spider): + request = Request(url, callback=spider.parse, dont_filter=True) + _add(domain, request) + # urls + for url in urls: + spider = spiders.fromurl(url) + if spider: + request = Request(url=url, callback=spider.parse, dont_filter=True) + _add(spider.domain_name, request) + else: + log.msg('Could not found spider for <%s>' % url, log.ERROR) + + # requests + for request in requests: + spider = spiders.fromurl(request.url) + if not spider: + log.msg('Could not found spider for %s' % request, log.ERROR) + continue + _add(spider.domain_name, request) + return perdomain + + def _start_urls(self, spider): + return spider.start_urls if hasattr(spider.start_urls, '__iter__') else [spider.start_urls] + +scrapymanager = ExecutionManager() diff --git a/scrapy/trunk/scrapy/core/prioritizers.py b/scrapy/trunk/scrapy/core/prioritizers.py new file mode 100644 index 000000000..214a475a3 --- /dev/null +++ b/scrapy/trunk/scrapy/core/prioritizers.py @@ -0,0 +1,45 @@ +""" +The Prioritizer is a class which receives a list of elements and prioritizes +it. It's used for defining the order in which domains are to be scraped. + +A Prioritizer basically consists of a class which receives a list of elements +in its constructs and contains only one method: get_priority() which returns +the priority of the given element. The element passed to get_priority() must +exists in the list of elements passed in the constructor. + +This module contains several basic prioritizers. + +For more advanced prioritizers see: scrapy.contrib.prioritizers +""" +import random + +class NullPrioritizer(object): + """ + This prioritizer always return the same priority (1) + """ + def __init__(self, elements): + pass + + def get_priority(self, element): + return 1 + +class RandomPrioritizer(object): + """ + This prioritizer always return a random priority + """ + def __init__(self, elements): + self.count = len(elements) + + def get_priority(self, element): + return random.randrange(0, self.count) + +class AlphabeticPrioritizer(object): + """ + This prioritizer priotizes based on the alphabetic order of the element + """ + def __init__(self, elements): + self.elements = elements[:] + self.elements.sort() + + def get_priority(self, element): + return self.elements.index(element) diff --git a/scrapy/trunk/scrapy/core/scheduler/__init__.py b/scrapy/trunk/scrapy/core/scheduler/__init__.py new file mode 100644 index 000000000..1027c0382 --- /dev/null +++ b/scrapy/trunk/scrapy/core/scheduler/__init__.py @@ -0,0 +1,3 @@ +from scrapy.core.scheduler.schedulers import Scheduler +from scrapy.core.scheduler.filter import GroupFilter +from scrapy.core.scheduler.store import MemoryStore diff --git a/scrapy/trunk/scrapy/core/scheduler/filter.py b/scrapy/trunk/scrapy/core/scheduler/filter.py new file mode 100644 index 000000000..9008a4d03 --- /dev/null +++ b/scrapy/trunk/scrapy/core/scheduler/filter.py @@ -0,0 +1,19 @@ +class GroupFilter(dict): + """Filter groups of keys""" + def open(self, group): + self[group] = set() + + def close(self, group): + del self[group] + + def add(self, group, key): + """Add a key to the group if an equivalent key has not already been added. + This method will return true if the key was added and false otherwise. + """ + if key not in self[group]: + self[group].add(key) + return True + return False + + def has(self, group, key): + return key in self[group] diff --git a/scrapy/trunk/scrapy/core/scheduler/schedulers.py b/scrapy/trunk/scrapy/core/scheduler/schedulers.py new file mode 100644 index 000000000..40cf7b5ef --- /dev/null +++ b/scrapy/trunk/scrapy/core/scheduler/schedulers.py @@ -0,0 +1,173 @@ +""" +The Scrapy Scheduler +""" + +from twisted.internet import defer + +from scrapy.core.scheduler.filter import GroupFilter +from scrapy.core import log +from scrapy.core.exceptions import IgnoreRequest +from scrapy.utils.datatypes import PriorityQueue, PriorityStack +from scrapy.utils.misc import defer_fail +from scrapy.conf import settings + + +class Scheduler(object) : + """ + The scheduler decides what to scrape, how fast, and in what order. + The scheduler schedules websites and pages to be scraped. Individual + web pages that are to be scraped are batched up into a "run" for a website. + As the domain is being scraped, pages that are discovered are added to the + scheduler. The scheduler must not allow the same page to be requested + multiple times within the same batch. + + Typical usage: + + * next_availble_domain() called to find out when there is something to do + * begin_domain() called to commence scraping a website + * enqueue_request() called multiple times when new links found + * next_request() called multiple times when there is capacity to process urls + * close_domain() called when there are no more pages or upon error + + Note a couple things: + 1) The order in which you get back the list of pages to scrape is not + necesarily the order you put them in. + 2) A canonical URL is calculated for each url for each domain to check that + it is unique, however the actual url passed in is returned when + get_next_page is called. + This is for two main reasons: + * To be nice to the screen scraped site just incase the specific + format of the url is significant. + * To take advantage of any caching that uses the URL/URI as a key + + all_domains contains the names of all domains that are to be scheduled. + + Two crawling orders are available by default, which can be set with the + SCHEDULER_ORDER settings: + + * BFO - breath-first order (default). Consumes more memory than DFO but reaches + most relevant pages faster. + * DFO - depth-first order. Consumes less memory than BFO but usually takes + longer to reach the most relevant pages. + """ + + def __init__(self): + self.pending_domains_count = {} + self.domains_queue = PriorityQueue() + self.pending_requests = {} + self.groupfilter = GroupFilter() + self.dfo = settings.get('SCHEDULER_ORDER', '').upper() == 'DFO' + + def domain_is_open(self, domain): + return domain in self.pending_requests + + def is_pending(self, domain): + return domain in self.pending_domains_count + + def domain_has_pending(self, domain): + if domain in self.pending_requests: + return not self.pending_requests[domain].empty() + + def next_domain(self) : + """ + Return next domain available to scrape and remove it from available domains queue + """ + if self.pending_domains_count: + priority, domain = self.domains_queue.get_nowait() + if self.pending_domains_count[domain] == 1: + del self.pending_domains_count[domain] + else: + self.pending_domains_count[domain] -= 1 + return domain + return None + + def add_domain(self, domain, priority=1): + """ + This functions schedules a new domain to be scraped, with the given + priority. It doesn't check if the domain is already scheduled. A + domain can be scheduled twice, either with the same or with different + priority. + """ + self.domains_queue.put(domain, priority=priority) + if domain not in self.pending_domains_count: + self.pending_domains_count[domain] = 1 + else: + self.pending_domains_count[domain] += 1 + + def open_domain(self, domain): + """ + Allocates resources for maintaining a schedule for domain. + """ + if self.dfo: + self.pending_requests[domain] = PriorityStack() + else: + self.pending_requests[domain] = PriorityQueue() + + self.groupfilter.open(domain) + + def enqueue_request(self, domain, request, priority=1): + """ + Add a page to be scraped for a domain that is currently being scraped. + """ + requestid = request.fingerprint() + added = self.groupfilter.add(domain, requestid) + + if request.dont_filter or added: + deferred = defer.Deferred() + self.pending_requests[domain].put((request, deferred), priority) + return deferred + else: + return defer_fail(IgnoreRequest('Skipped (already visited): %s' % request)) + + def request_seen(self, domain, request): + """ + Returns True if the given Request was scheduled before for the given + domain + """ + return self.groupfilter.has(domain, request.fingerprint()) + + def next_request(self, domain): + """ + Get the next request to be scraped. + + None should be returned if there are no more request pending for the domain passed. + """ + pending_list = self.pending_requests.get(domain) + if pending_list and not pending_list.empty(): + return pending_list.get_nowait()[1] + else: + return (None, None) + + def close_domain(self, domain) : + """ + Called once we are finished scraping a domain. The scheduler will + free any resources associated with the domain. + """ + try : + del self.pending_requests[domain] + except Exception, inst: + msg = "Could not clear pending pages for domain %s, %s" % (domain, inst) + log.msg(msg, level=log.WARNING) + + try : + self.groupfilter.close(domain) + except Exception, inst: + msg = "Could not clear url filter for domain %s, %s" % (domain, inst) + log.msg(msg, level=log.WARNING) + + def remove_pending_domain(self, domain): + """ + Remove a pending domain not yet started. If the domain was enqueued + several times, all those instances are removed. + + Returns the number of times the domain was enqueued. 0 if domains was + not pending. + + If domain is open (not pending) it is not removed and returns None. You + need to call close_domain for open domains. + """ + if not self.domain_is_open(domain): + return self.pending_domains_count.pop(domain, 0) + + def is_idle(self): + return not self.pending_requests diff --git a/scrapy/trunk/scrapy/core/scheduler/store.py b/scrapy/trunk/scrapy/core/scheduler/store.py new file mode 100644 index 000000000..b7eddfcf4 --- /dev/null +++ b/scrapy/trunk/scrapy/core/scheduler/store.py @@ -0,0 +1,35 @@ +""" +The datastore module contains implementations of data storage engines for the +crawling process. These are used to track metrics on each site and on each +page visited. +""" +from datetime import datetime + +class MemoryStore(object) : + """Simple implementation of a data store. This is useful if no persistent + history is required (e.g. unit testing or development) and provides a + simpler reference implementation. + """ + def __init__(self) : + """The store will just be a dict with an entry for each site, that + entry will contain dicts and lists of data. + """ + self._store = {} + + def open(self, site): + self._store[site] = {} + + def close_site(self, site): + del self._store[site] + + def store(self, site, key, url, parent=None, version=None, post_version=None): + checked = datetime.now() + self._store[key] = (version, checked) + + def status(self, site, key): + """Get the version and last checked time for a key. + + this will be changed later to support checking (lazily) the last modified + time and perhaps other statistics needed by the scheduling algorithms + """ + return self._store.get(key) diff --git a/scrapy/trunk/scrapy/core/signals.py b/scrapy/trunk/scrapy/core/signals.py new file mode 100644 index 000000000..658cf70c4 --- /dev/null +++ b/scrapy/trunk/scrapy/core/signals.py @@ -0,0 +1,69 @@ +""" +Scrapy core signals +""" +from pydispatch import dispatcher + +from scrapy.core import log + +# After the execution engine has started +# args: None +engine_started = object() + +# After the execution engine has stopped +# args: None +engine_stopped = object() + +# Before opening a new domain for crawling +# args: domain, spider +domain_open = object() + +# After a domain has been opened for crawling +# args: domain, spider +domain_opened = object() + +# When a domain has no remaining requests to process +# args: domain, spider +domain_idle = object() + +# After a domain has been closed +# args: domain, spider, status +# status is a string and its possible values are: "finished" or "cancelled" +domain_closed = object() + +# domain has been initialized (successful or not) +# args: domain, spider, status +domain_initialized = object() + +# New request received from spiders +# args: request, spider, response +# response is the response (fed to the spider) which generated the request +request_received = object() + +# When request is sent in the downloader +# args: request, spider +request_uploaded = object() + +# When response arrives from the downloader +# args: response, spider +response_downloaded = object() + +# After item is returned from spiders +# args: item, spider, response +item_scraped = object() + +# After item is processed by pipeline and pass all its stages +# args: item, spider, response, pipe_output +item_passed = object() + +# After item is dropped by pipeline +# args: item, spider, response, exception +item_dropped = object() + +def send_catch_log(signal, sender=None, **kwargs): + """ + Send a signal and log any exceptions raised by its listeners + """ + try: + dispatcher.send(signal=signal, sender=sender, **kwargs) + except: + log.exc("Exception catched on signal dispatch") diff --git a/scrapy/trunk/scrapy/extension/__init__.py b/scrapy/trunk/scrapy/extension/__init__.py new file mode 100644 index 000000000..63cc43c52 --- /dev/null +++ b/scrapy/trunk/scrapy/extension/__init__.py @@ -0,0 +1,37 @@ +""" +This module contains the ExtensionManager which takes care of loading and +keeping track of all enabled extensions. It also contains an instantiated +ExtensionManager (extensions) to be used as singleton. +""" +from scrapy.core.exceptions import NotConfigured +from scrapy.utils.misc import load_class +from scrapy.conf import settings + +class ExtensionManager(object): + + def __init__(self): + self.loaded = False + self.enabled = {} + self.disabled = {} + + def load(self): + """ + Load enabled extensions in settings module + """ + + self.loaded = False + self.enabled.clear() + self.disabled.clear() + for extension_path in settings.getlist('EXTENSIONS'): + try: + cls = load_class(extension_path) + self.enabled[cls.__name__] = cls() + except NotConfigured: + self.disabled[cls.__name__] = extension_path + pass + self.loaded = True + + def reload(self): + self.load() + +extensions = ExtensionManager() diff --git a/scrapy/trunk/scrapy/fetcher/__init__.py b/scrapy/trunk/scrapy/fetcher/__init__.py new file mode 100644 index 000000000..29fbde74a --- /dev/null +++ b/scrapy/trunk/scrapy/fetcher/__init__.py @@ -0,0 +1,51 @@ +import urlparse + +from scrapy.spider import spiders +from scrapy.http import Request +from scrapy.core.manager import scrapymanager +from scrapy.spider import BaseSpider + +def fetch(urls, perdomain=False): + """Download a set of urls. This is starts a reactor and is suitable + for calling from a main method - not for calling from within the + framework. + + It will return a list of Response objects for all pages successfully + downloaded. + """ + bigbucket = {} # big bucket to store response objects (per domain) + + def _add(response, domain): + if not domain in bigbucket: + bigbucket[domain] = [] + bigbucket[domain] += [response] + + # url clasification by domain + requestdata = set() + for url in urls: + domain = get_or_create_spider(url).domain_name + request = Request(url=url, callback=lambda r: _add(r, domain), dont_filter=True) + requestdata.add(request) + + scrapymanager.runonce(*requestdata) + + # returns bigbucket as dict or flatted + if perdomain: + return bigbucket + + flatbucket = [] + for domain, responses in bigbucket.iteritems(): + flatbucket.extend(responses) + return flatbucket + + +def get_or_create_spider(url): + # dirty hack to allow downloading pages from unknown domains + spider = spiders.fromurl(url) + if not spider: + domain = str(urlparse.urlparse(url).hostname or spiders.default_domain) + spider = BaseSpider() + spider.domain_name = domain + spiders._alldict[domain] = spider + return spider + diff --git a/scrapy/trunk/scrapy/http/__init__.py b/scrapy/trunk/scrapy/http/__init__.py new file mode 100644 index 000000000..d9fef2626 --- /dev/null +++ b/scrapy/trunk/scrapy/http/__init__.py @@ -0,0 +1,11 @@ +""" +Module containing all HTTP related classes + +Use this module (instead of the more specific ones) when importing Headers, +Request, Response, ResponseBody and Url outside this module. +""" + +from scrapy.http.url import Url +from scrapy.http.headers import Headers +from scrapy.http.request import Request +from scrapy.http.response import Response, ResponseBody diff --git a/scrapy/trunk/scrapy/http/headers.py b/scrapy/trunk/scrapy/http/headers.py new file mode 100644 index 000000000..54d74a9ef --- /dev/null +++ b/scrapy/trunk/scrapy/http/headers.py @@ -0,0 +1,97 @@ +from scrapy.utils.datatypes import CaselessDict + +def headers_raw_to_dict(headers_raw): + """ + Convert raw headers (single multi-line string) + to the dictionary. + + For example: + >>> headers_raw_to_dict("Content-type: text/html\\n\\rAccept: gzip\\n\\n") + {'Content-type': ['text/html'], 'Accept': ['gzip']} + + Incorrect input: + >>> headers_raw_to_dict("Content-typt gzip\\n\\n") + {} + + Argument is None: + >>> headers_raw_to_dict(None) + """ + if headers_raw is None: + return None + return dict([ + (header_item[0].strip(), [header_item[1].strip()]) + for header_item + in [ + header.split(':', 1) + for header + in headers_raw.splitlines()] + if len(header_item) == 2]) + +def headers_dict_to_raw(headers_dict): + """ + Returns a raw HTTP headers representation of headers + + For example: + >>> headers_dict_to_raw({'Content-type': 'text/html', 'Accept': 'gzip'}) + 'Content-type: text/html\\r\\nAccept: gzip' + >>> from twisted.python.util import InsensitiveDict + >>> td = InsensitiveDict({'Content-type': ['text/html'], 'Accept': ['gzip']}) + >>> headers_dict_to_raw(td) + 'Content-type: text/html\\r\\nAccept: gzip' + + Argument is None: + >>> headers_dict_to_raw(None) + + """ + if headers_dict is None: + return None + raw_lines = [] + for key, value in headers_dict.items(): + if isinstance(value, (str, unicode)): + raw_lines.append("%s: %s" % (key, value)) + elif isinstance(value, (list, tuple)): + for v in value: + raw_lines.append("%s: %s" % (key, v)) + return '\r\n'.join(raw_lines) + +class Headers(CaselessDict): + def __init__(self, dictorstr=None, fromdict=None, fromstr=None, encoding='utf-8'): + self.encoding = encoding + + if dictorstr is not None: + if isinstance(dictorstr, dict): + d = dictorstr + elif isinstance(dictorstr, basestring): + d = headers_raw_to_dict(dictorstr) + elif fromdict is not None: + d = fromdict + elif fromstr is not None: + d = headers_raw_to_dict(fromstr) + else: + d = {} + + # can't use CaselessDict.__init__(self, d) because it doesn't call __setitem__ + for k,v in d.iteritems(): + self.__setitem__(k.lower(), v) + + def normkey(self, key): + return key.title() + + def __setitem__(self, key, value): + """Headers must not be unicode""" + if isinstance(key, unicode): + key = key.encode(self.encoding) + if isinstance(value, unicode): + value = value.encode(self.encoding) + super(Headers, self).__setitem__(key, value) + + def tostring(self): + return headers_dict_to_raw(self) + + def rawsize(self): + """Estimated size of raw HTTP headers, in bytes""" + # For each header line you have 4 extra chars: ": " and CRLF + return sum([len(k)+len(v)+4 for k, v in self.iteritems()]) + + def to_string(self): + return headers_dict_to_raw(self) diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py new file mode 100644 index 000000000..2c0623ed1 --- /dev/null +++ b/scrapy/trunk/scrapy/http/request.py @@ -0,0 +1,180 @@ +import urllib +import warnings +from sha import sha +from copy import copy +from base64 import urlsafe_b64encode + +from twisted.internet import defer + +from scrapy.http.url import Url +from scrapy.http.headers import Headers +from scrapy.utils.url import safe_url_string +from scrapy.utils.c14n import canonicalize +from scrapy.utils.misc import chain_deferred + +class Request(object): + def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None, + referer=None, url_encoding='utf-8', link_text='', http_user='', http_pass='', dont_filter=None, + fingerprint_params=None): + + self.encoding = url_encoding # this one has to be set first + self.set_url(url) + + # method + if method is None and body is not None: + method = 'POST' # backwards compatibility + self.method = method.upper() if method else 'GET' + assert isinstance(self.method, basestring), 'Request method argument must be str or unicode, got %s: %s' % (type(method), method) + + # body + if isinstance(body, dict): + body = urllib.urlencode(body) + self.body = body + + # callback / deferred + if callable(callback): + callback = defer.Deferred().addCallback(callback) + self.deferred = callback or defer.Deferred() + + # request cookies + self.cookies = cookies or {} + # request headers + self.headers = Headers(headers or {}, encoding=url_encoding) + # persistent context across requests + self.context = context or {} + # dont_filter be filtered by scheduler + self.dont_filter = dont_filter + # fingerprint parameters + self.fingerprint_params = fingerprint_params or {} + self._fingerprint = None + # shortcut for setting referer + if referer is not None: + self.headers['referer'] = referer + # http auth + if http_user or http_pass: + self.httpauth(http_user, http_pass) + self.depth = 0 + self.link_text = link_text + + def append_callback(self, callback, *args, **kwargs): + if isinstance(callback, defer.Deferred): + return chain_deferred(self.deferred, callback) + return self.deferred.addCallback(callback, *args, **kwargs) + + def prepend_callback(self, func, *args, **kwargs): + if callable(func): + func = defer.Deferred().addCallback(func, *args, **kwargs) + assert isinstance(func, defer.Deferred), 'prepend_callback expects a callable or defer.Deferred instance, got %s' % type(func) + self.deferred = chain_deferred(func, self.deferred) + return self.deferred + + def set_url(self, url): + assert isinstance(url, basestring), 'Request url argument must be str or unicode, got %s:' % (type(url), url) + decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding) + self._url = Url(safe_url_string(decoded_url, self.encoding)) + url = property(lambda x: x._url, set_url) + + def httpauth(self, http_user, http_pass): + if not http_user: + http_user = '' + if not http_pass: + http_pass = '' + self.headers['Authorization'] = 'Basic ' + urlsafe_b64encode("%s:%s" % (http_user, http_pass)) + + def __getitem__(self, key): + warnings.warn("Request tuples have been deprecated, Request objects are now used. Index mappings: 0 -> r.url, 1 -> r.body, 2 -> r.deferred, 3 -> r.options ", DeprecationWarning) + return (self.url, self.body, self.deferred, {})[key] + + def __str__(self): + if self.method != 'GET': + return "<(%s) %s>" % (self.method, self.url) + return "<%s>" % self.url + + def __len__(self): + """Return raw HTTP request size""" + return len(self.to_string()) + + def traceinfo(self): + fp = self.fingerprint() + version = '%s..%s' % (fp[:4], fp[-4:]) + return "" % (self.method, self.url, version) + + def __repr__(self): + d = { + 'method': self.method, + 'url': self.url, + 'headers': self.headers, + 'cookies': self.cookies, + 'body': self.body, + 'context': self.context + } + return "%s(%s)" % (self.__class__.__name__, repr(d)) + + def copy(self): + """Clone request except `context` attribute""" + new = copy(self) + for att in self.__dict__: + if att not in ['context', 'url', 'deferred', '_fingerprint']: + value = getattr(self, att) + setattr(new, att, copy(value)) + new.deferred = defer.Deferred() + new.context = self.context # requests shares same context dictionary + new._fingerprint = None # reset fingerprint + return new + + def fingerprint(self): + """Returns unique resource fingerprint""" + if self._fingerprint and not self.fingerprint_params: + return self._fingerprint + + headers = {} + if self.fingerprint_params: + if 'tamperfunc' in self.fingerprint_params: + tamperfunc = self.fingerprint_params['tamperfunc'] + assert callable(tamperfunc) + req = tamperfunc(self.copy()) + assert isinstance(req, Request) + try: + del req.fingerprint_params['tamperfunc'] + except KeyError: + pass + return req.fingerprint() + + if self.headers: + if 'include_headers' in self.fingerprint_params: + keys = [k.lower() for k in self.fingerprint_params['include_headers']] + headers = dict([(k, v) for k, v in self.headers.items() if k.lower() in keys]) + elif 'exclude_headers' in self.fingerprint_params: + keys = [k.lower() for k in self.fingerprint_params['exclude_headers']] + headers = dict([(k, v) for k, v in self.headers.items() if k.lower() not in keys]) + + # fingerprint generation + fp = sha(canonicalize(self.url)) + fp.update(self.method) + + if self.body and self.method in ['POST', 'PUT']: + fp.update(self.body) + + if headers: + for k, v in sorted([(k.lower(), v) for k, v in headers.items()]): + fp.update(k) + fp.update(v) + + self._fingerprint = fp.hexdigest() + return self._fingerprint + + def to_string(self): + """ Return raw HTTP request representation (as string). This is + provided only for reference since it's not the actual stream of bytes + that will be send when performing the request (that's controlled by + Twisted). + """ + + s = "%s %s HTTP/1.1\r\n" % (self.method, self.url) + s += "Host: %s\r\n" % self.url.hostname + s += self.headers.to_string() + "\r\n" + s += "\r\n" + if self.body: + s += self.body + s += "\r\n" + return s diff --git a/scrapy/trunk/scrapy/http/response.py b/scrapy/trunk/scrapy/http/response.py new file mode 100644 index 000000000..742486e88 --- /dev/null +++ b/scrapy/trunk/scrapy/http/response.py @@ -0,0 +1,186 @@ +import re +import sha +import copy + +from BeautifulSoup import UnicodeDammit + +from scrapy.http.url import Url +from scrapy.http.headers import Headers + +from twisted.web import http +reason_phrases = http.RESPONSES + +class Response(object) : + """HTTP responses + + Arguments: + * Domain - the spider domain for the page + * url - the final url for the resource + * original_url - the url requested + * headers - HTTP headers + * status - HTTP status code + * body - Body object containing the content of the response + * parent - the URL of the referring page + """ + _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) + + def __init__(self, domain, url, original_url=None, headers=None, status=200, body=None, parent=None): + self.domain = domain + self.url = Url(url) + self.original_url = Url(original_url) if original_url else url # different if redirected or escaped + self.headers = Headers(headers or {}) + self.status = status + if isinstance(body, str): + self.body = ResponseBody(body, self.headers_encoding()) + else: + self.body = body + self.parent = parent + self.cached = False + self.request = None # request which originated this response + + def version(self): + """A hash of the contents of this response""" + if not hasattr(self, '_version'): + self._version = sha.new(self.body.to_string()).hexdigest() + return self._version + + def headers_encoding(self): + content_type = self.headers.get('Content-Type') + if content_type: + encoding = self._ENCODING_RE.search(content_type[0]) + if encoding: + return encoding.group(1) + + def __repr__(self): + return "Response(domain=%s, url=%s, original_url=%s, headers=%s, status=%s, body=%s, parent=%s)" % \ + (repr(self.domain), repr(self.url), repr(self.original_url), repr(self.headers), repr(self.status), repr(self.body), repr(self.parent)) + + def __str__(self): + version = '%s..%s' % (self.version()[:4], self.version()[-4:]) + return "" % (self.status, self.url, version) + + def __len__(self): + """Return raw HTTP response size""" + return len(self.to_string()) + + def info(self): + return "[^;]+);\s*charset=(?P[\w-]+)', re.I) + + def __init__(self, content, declared_encoding=None): + self._content = content + self._unicode_content = None + self.declared_encoding = declared_encoding + self._expected_encoding = None + self._actual_encoding = None + + def to_string(self, encoding=None): + """Get the body as a string. If an encoding is specified, the + body will be encoded using that encoding. + """ + if encoding in (None, self.declared_encoding, self._actual_encoding): + return self._content + # should we cache this decode? + return self.to_unicode().encode(encoding) + + def to_unicode(self): + """Return body as unicode string""" + if self._unicode_content: + return self._unicode_content + proposed = self.get_expected_encoding() + dammit = UnicodeDammit(self._content, [proposed]) + self._actual_encoding = dammit.originalEncoding + self._unicode_content = dammit.unicode + return dammit.unicode + + def get_content(self): + """Return original content bytes""" + return self._content + + def get_declared_encoding(self): + """Get the value of the declared encoding passed to the + constructor. + """ + return self.declared_encoding + + def get_real_encoding(self): + """Get the real encoding, by trying first the expected and then the + actual encoding. + """ + if self.get_expected_encoding(): + result = self.get_expected_encoding() + else: + self.to_unicode() + result = self._actual_encoding + return result + + def get_expected_encoding(self): + """Get the expected encoding for the page. This is the declared + encoding, or the meta tag encoding. + """ + if self._expected_encoding: + return self._expected_encoding + proposed = self.declared_encoding + if not proposed: + match = self.CHARSET_RE.search(self._content[:5000]) + if match: + proposed = match.group("charset") + self._expected_encoding = proposed + return proposed + + def __repr__(self): + return "ResponseBody(content=%s, declared_encoding=%s)" % (repr(self._content), repr(self.declared_encoding)) + + def __str__(self): + return self.to_string() + + def __unicode__(self): + return self.to_unicode() + + def __len__(self): + return len(self._content) + diff --git a/scrapy/trunk/scrapy/http/url.py b/scrapy/trunk/scrapy/http/url.py new file mode 100644 index 000000000..098150be1 --- /dev/null +++ b/scrapy/trunk/scrapy/http/url.py @@ -0,0 +1,57 @@ +""" +The Url class is similar to the urlparse.ParseResult class (it has the same +attributes) with the following differences: + +- it inherits from str +- it's lazy, so it only parses the url when needed +""" + +import urlparse + +class Url(str): + + @property + def parsedurl(self): + if not hasattr(self, '_parsedurl'): + self._parsedurl = urlparse.urlparse(self) + return self._parsedurl + + @property + def scheme(self): + return self.parsedurl.scheme + + @property + def netloc(self): + return self.parsedurl.netloc + + @property + def path(self): + return self.parsedurl.path + + @property + def params(self): + return self.parsedurl.params + + @property + def query(self): + return self.parsedurl.query + + @property + def fragment(self): + return self.parsedurl.fragment + + @property + def username(self): + return self.parsedurl.username + + @property + def password(self): + return self.parsedurl.password + + @property + def hostname(self): + return self.parsedurl.hostname + + @property + def port(self): + return self.parsedurl.port diff --git a/scrapy/trunk/scrapy/item/__init__.py b/scrapy/trunk/scrapy/item/__init__.py new file mode 100644 index 000000000..35982e691 --- /dev/null +++ b/scrapy/trunk/scrapy/item/__init__.py @@ -0,0 +1 @@ +from scrapy.item.models import ScrapedItem diff --git a/scrapy/trunk/scrapy/item/models.py b/scrapy/trunk/scrapy/item/models.py new file mode 100644 index 000000000..ea99cc43a --- /dev/null +++ b/scrapy/trunk/scrapy/item/models.py @@ -0,0 +1,16 @@ +class ScrapedItem(object): + """ + This is the base class for all scraped items. + + The only required attributes are: + * guid (unique global indentifier) + * url (URL where that item was scraped from) + """ + + def assign(self, name, value): + """Assign an attribute. Can receive a Selector in value which will + extract its data """ + + if hasattr(value, 'extract'): + value = value.extract() + setattr(self, name, value) diff --git a/scrapy/trunk/scrapy/item/pipeline.py b/scrapy/trunk/scrapy/item/pipeline.py new file mode 100644 index 000000000..ef2251a3b --- /dev/null +++ b/scrapy/trunk/scrapy/item/pipeline.py @@ -0,0 +1,100 @@ +from scrapy.core import log +from scrapy.core.exceptions import DropItem, NotConfigured +from scrapy.item import ScrapedItem +from scrapy.utils.misc import load_class, defer_succeed, mustbe_deferred +from scrapy.conf import settings + +class ItemPipeline(object): + + def __init__(self): + self.loaded = False + self.pipeline = [] + self.domaininfo = {} + self.load() + + def load(self): + """ + Load pipelines stages defined in settings module + """ + for stage in settings.getlist('ITEM_PIPELINES') or (): + cls = load_class(stage) + if cls: + try: + stageinstance = cls() + self.pipeline.append(stageinstance) + except NotConfigured: + pass + log.msg("Enabled item pipelines: %s" % ", ".join([type(p).__name__ for p in self.pipeline])) + self.loaded = True + + def open_domain(self, domain): + self.domaininfo[domain] = set() + for pipe in self.pipeline: + if hasattr(pipe, 'open_domain'): + pipe.open_domain(domain) + + def close_domain(self, domain): + for pipe in self.pipeline: + if hasattr(pipe, 'close_domain'): + pipe.close_domain(domain) + del self.domaininfo[domain] + + def is_idle(self): + return not self.domaininfo + + def domain_is_idle(self, domain): + return not self.domaininfo.get(domain) + + def pipe(self, item, spider, response): + """ + item pipelines are instanceable classes that defines a `pipeline` method + that takes ScrapedItem as input and returns ScrapedItem. + + The output from one stage is the input of the next. + + Raising DropItem stops pipeline. + + This pipeline is configurable with the ITEM_PIPELINES setting + """ + if not self.pipeline: + return defer_succeed(item) + + domain = spider.domain_name + pipeline = self.pipeline[:] + current_stage = pipeline[0] + info = self.domaininfo[domain] + info.add(item) + + def _next_stage(item): + assert isinstance(item, ScrapedItem), 'Pipeline stages must return a ScrapedItem or raise DropItem' + if not pipeline: + return item + + current_stage = pipeline.pop(0) + log.msg("_%s_ Pipeline stage: %s" % (item.guid, type(current_stage).__name__), log.TRACE, domain=domain) + + d = mustbe_deferred(current_stage.process_item, domain, response, item) + d.addCallback(_next_stage) + return d + + def _ondrop(_failure): + ex = _failure.value + if isinstance(ex, DropItem): + # TODO: current_stage is not working, check why + #log.msg("%s: Dropped %s - %s" % (type(current_stage).__name__, item, str(ex)), log.DEBUG, domain=domain) + log.msg("Dropped %s - %s" % (item, str(ex)), log.DEBUG, domain=domain) + return _failure + else: + # TODO: current_stage is not working, check why + #log.msg('%s: Error processing %s - %s' % (type(current_stage).__name__, item, _failure), log.ERROR, domain=domain) + log.msg('Error processing %s - %s' % (item, _failure), log.ERROR, domain=domain) + + def _pipeline_finished(_): + log.msg("_%s_ Pipeline finished" % item.guid, log.TRACE, domain=domain) + info.remove(item) + return _ + + deferred = mustbe_deferred(_next_stage, item) + deferred.addErrback(_ondrop) + deferred.addBoth(_pipeline_finished) + return deferred diff --git a/scrapy/trunk/scrapy/lib/BeautifulSoup.py b/scrapy/trunk/scrapy/lib/BeautifulSoup.py new file mode 100644 index 000000000..0e55abaf8 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/BeautifulSoup.py @@ -0,0 +1,1931 @@ +"""Beautiful Soup +Elixir and Tonic +"The Screen-Scraper's Friend" +http://www.crummy.com/software/BeautifulSoup/ + +Beautiful Soup parses a (possibly invalid) XML or HTML document into a +tree representation. It provides methods and Pythonic idioms that make +it easy to navigate, search, and modify the tree. + +A well-formed XML/HTML document yields a well-formed data +structure. An ill-formed XML/HTML document yields a correspondingly +ill-formed data structure. If your document is only locally +well-formed, you can use this library to find and process the +well-formed part of it. + +Beautiful Soup works with Python 2.2 and up. It has no external +dependencies, but you'll have more success at converting data to UTF-8 +if you also install these three packages: + +* chardet, for auto-detecting character encodings + http://chardet.feedparser.org/ +* cjkcodecs and iconv_codec, which add more encodings to the ones supported + by stock Python. + http://cjkpython.i18n.org/ + +Beautiful Soup defines classes for two main parsing strategies: + + * BeautifulStoneSoup, for parsing XML, SGML, or your domain-specific + language that kind of looks like XML. + + * BeautifulSoup, for parsing run-of-the-mill HTML code, be it valid + or invalid. This class has web browser-like heuristics for + obtaining a sensible parse tree in the face of common HTML errors. + +Beautiful Soup also defines a class (UnicodeDammit) for autodetecting +the encoding of an HTML or XML document, and converting it to +Unicode. Much of this code is taken from Mark Pilgrim's Universal Feed Parser. + +For more than you ever wanted to know about Beautiful Soup, see the +documentation: +http://www.crummy.com/software/BeautifulSoup/documentation.html + +Here, have some legalese: + +Copyright (c) 2004-2007, Leonard Richardson + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the the Beautiful Soup Consortium and All + Night Kosher Bakery nor the names of its contributors may be + used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE, DAMMIT. + +""" +from __future__ import generators + +__author__ = "Leonard Richardson (leonardr@segfault.org)" +__version__ = "3.0.6" +__copyright__ = "Copyright (c) 2004-2008 Leonard Richardson" +__license__ = "New-style BSD" + +from sgmllib import SGMLParser, SGMLParseError +import codecs +import types +import re +import sgmllib +try: + from htmlentitydefs import name2codepoint +except ImportError: + name2codepoint = {} + +#This hack makes Beautiful Soup able to parse XML with namespaces +sgmllib.tagfind = re.compile('[a-zA-Z][-_.:a-zA-Z0-9]*') + +DEFAULT_OUTPUT_ENCODING = "utf-8" + +# First, the classes that represent markup elements. + +class PageElement: + """Contains the navigational information for some part of the page + (either a tag or a piece of text)""" + + def setup(self, parent=None, previous=None): + """Sets up the initial relations between this element and + other elements.""" + self.parent = parent + self.previous = previous + self.next = None + self.previousSibling = None + self.nextSibling = None + if self.parent and self.parent.contents: + self.previousSibling = self.parent.contents[-1] + self.previousSibling.nextSibling = self + + def replaceWith(self, replaceWith): + oldParent = self.parent + myIndex = self.parent.contents.index(self) + if hasattr(replaceWith, 'parent') and replaceWith.parent == self.parent: + # We're replacing this element with one of its siblings. + index = self.parent.contents.index(replaceWith) + if index and index < myIndex: + # Furthermore, it comes before this element. That + # means that when we extract it, the index of this + # element will change. + myIndex = myIndex - 1 + self.extract() + oldParent.insert(myIndex, replaceWith) + + def extract(self): + """Destructively rips this element out of the tree.""" + if self.parent: + try: + self.parent.contents.remove(self) + except ValueError: + pass + + #Find the two elements that would be next to each other if + #this element (and any children) hadn't been parsed. Connect + #the two. + lastChild = self._lastRecursiveChild() + nextElement = lastChild.next + + if self.previous: + self.previous.next = nextElement + if nextElement: + nextElement.previous = self.previous + self.previous = None + lastChild.next = None + + self.parent = None + if self.previousSibling: + self.previousSibling.nextSibling = self.nextSibling + if self.nextSibling: + self.nextSibling.previousSibling = self.previousSibling + self.previousSibling = self.nextSibling = None + return self + + def _lastRecursiveChild(self): + "Finds the last element beneath this object to be parsed." + lastChild = self + while hasattr(lastChild, 'contents') and lastChild.contents: + lastChild = lastChild.contents[-1] + return lastChild + + def insert(self, position, newChild): + if (isinstance(newChild, basestring) + or isinstance(newChild, unicode)) \ + and not isinstance(newChild, NavigableString): + newChild = NavigableString(newChild) + + position = min(position, len(self.contents)) + if hasattr(newChild, 'parent') and newChild.parent != None: + # We're 'inserting' an element that's already one + # of this object's children. + if newChild.parent == self: + index = self.find(newChild) + if index and index < position: + # Furthermore we're moving it further down the + # list of this object's children. That means that + # when we extract this element, our target index + # will jump down one. + position = position - 1 + newChild.extract() + + newChild.parent = self + previousChild = None + if position == 0: + newChild.previousSibling = None + newChild.previous = self + else: + previousChild = self.contents[position-1] + newChild.previousSibling = previousChild + newChild.previousSibling.nextSibling = newChild + newChild.previous = previousChild._lastRecursiveChild() + if newChild.previous: + newChild.previous.next = newChild + + newChildsLastElement = newChild._lastRecursiveChild() + + if position >= len(self.contents): + newChild.nextSibling = None + + parent = self + parentsNextSibling = None + while not parentsNextSibling: + parentsNextSibling = parent.nextSibling + parent = parent.parent + if not parent: # This is the last element in the document. + break + if parentsNextSibling: + newChildsLastElement.next = parentsNextSibling + else: + newChildsLastElement.next = None + else: + nextChild = self.contents[position] + newChild.nextSibling = nextChild + if newChild.nextSibling: + newChild.nextSibling.previousSibling = newChild + newChildsLastElement.next = nextChild + + if newChildsLastElement.next: + newChildsLastElement.next.previous = newChildsLastElement + self.contents.insert(position, newChild) + + def append(self, tag): + """Appends the given tag to the contents of this tag.""" + self.insert(len(self.contents), tag) + + def findNext(self, name=None, attrs={}, text=None, **kwargs): + """Returns the first item that matches the given criteria and + appears after this Tag in the document.""" + return self._findOne(self.findAllNext, name, attrs, text, **kwargs) + + def findAllNext(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns all items that match the given criteria and appear + after this Tag in the document.""" + return self._findAll(name, attrs, text, limit, self.nextGenerator, + **kwargs) + + def findNextSibling(self, name=None, attrs={}, text=None, **kwargs): + """Returns the closest sibling to this Tag that matches the + given criteria and appears after this Tag in the document.""" + return self._findOne(self.findNextSiblings, name, attrs, text, + **kwargs) + + def findNextSiblings(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns the siblings of this Tag that match the given + criteria and appear after this Tag in the document.""" + return self._findAll(name, attrs, text, limit, + self.nextSiblingGenerator, **kwargs) + fetchNextSiblings = findNextSiblings # Compatibility with pre-3.x + + def findPrevious(self, name=None, attrs={}, text=None, **kwargs): + """Returns the first item that matches the given criteria and + appears before this Tag in the document.""" + return self._findOne(self.findAllPrevious, name, attrs, text, **kwargs) + + def findAllPrevious(self, name=None, attrs={}, text=None, limit=None, + **kwargs): + """Returns all items that match the given criteria and appear + before this Tag in the document.""" + return self._findAll(name, attrs, text, limit, self.previousGenerator, + **kwargs) + fetchPrevious = findAllPrevious # Compatibility with pre-3.x + + def findPreviousSibling(self, name=None, attrs={}, text=None, **kwargs): + """Returns the closest sibling to this Tag that matches the + given criteria and appears before this Tag in the document.""" + return self._findOne(self.findPreviousSiblings, name, attrs, text, + **kwargs) + + def findPreviousSiblings(self, name=None, attrs={}, text=None, + limit=None, **kwargs): + """Returns the siblings of this Tag that match the given + criteria and appear before this Tag in the document.""" + return self._findAll(name, attrs, text, limit, + self.previousSiblingGenerator, **kwargs) + fetchPreviousSiblings = findPreviousSiblings # Compatibility with pre-3.x + + def findParent(self, name=None, attrs={}, **kwargs): + """Returns the closest parent of this Tag that matches the given + criteria.""" + # NOTE: We can't use _findOne because findParents takes a different + # set of arguments. + r = None + l = self.findParents(name, attrs, 1) + if l: + r = l[0] + return r + + def findParents(self, name=None, attrs={}, limit=None, **kwargs): + """Returns the parents of this Tag that match the given + criteria.""" + + return self._findAll(name, attrs, None, limit, self.parentGenerator, + **kwargs) + fetchParents = findParents # Compatibility with pre-3.x + + #These methods do the real heavy lifting. + + def _findOne(self, method, name, attrs, text, **kwargs): + r = None + l = method(name, attrs, text, 1, **kwargs) + if l: + r = l[0] + return r + + def _findAll(self, name, attrs, text, limit, generator, **kwargs): + "Iterates over a generator looking for things that match." + + if isinstance(name, SoupStrainer): + strainer = name + else: + # Build a SoupStrainer + strainer = SoupStrainer(name, attrs, text, **kwargs) + results = ResultSet(strainer) + g = generator() + while True: + try: + i = g.next() + except StopIteration: + break + if i: + found = strainer.search(i) + if found: + results.append(found) + if limit and len(results) >= limit: + break + return results + + #These Generators can be used to navigate starting from both + #NavigableStrings and Tags. + def nextGenerator(self): + i = self + while i: + i = i.next + yield i + + def nextSiblingGenerator(self): + i = self + while i: + i = i.nextSibling + yield i + + def previousGenerator(self): + i = self + while i: + i = i.previous + yield i + + def previousSiblingGenerator(self): + i = self + while i: + i = i.previousSibling + yield i + + def parentGenerator(self): + i = self + while i: + i = i.parent + yield i + + # Utility methods + def substituteEncoding(self, str, encoding=None): + encoding = encoding or "utf-8" + return str.replace("%SOUP-ENCODING%", encoding) + + def toEncoding(self, s, encoding=None): + """Encodes an object to a string in some encoding, or to Unicode. + .""" + if isinstance(s, unicode): + if encoding: + s = s.encode(encoding) + elif isinstance(s, str): + if encoding: + s = s.encode(encoding) + else: + s = unicode(s) + else: + if encoding: + s = self.toEncoding(str(s), encoding) + else: + s = unicode(s) + return s + +class NavigableString(unicode, PageElement): + + def __getnewargs__(self): + return (NavigableString.__str__(self),) + + def __getattr__(self, attr): + """text.string gives you text. This is for backwards + compatibility for Navigable*String, but for CData* it lets you + get the string without the CData wrapper.""" + if attr == 'string': + return self + else: + raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__.__name__, attr) + + def __unicode__(self): + return str(self).decode(DEFAULT_OUTPUT_ENCODING) + + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + if encoding: + return self.encode(encoding) + else: + return self + +class CData(NavigableString): + + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + return "" % NavigableString.__str__(self, encoding) + +class ProcessingInstruction(NavigableString): + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + output = self + if "%SOUP-ENCODING%" in output: + output = self.substituteEncoding(output, encoding) + return "" % self.toEncoding(output, encoding) + +class Comment(NavigableString): + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + return "" % NavigableString.__str__(self, encoding) + +class Declaration(NavigableString): + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING): + return "" % NavigableString.__str__(self, encoding) + +class Tag(PageElement): + + """Represents a found HTML tag with its attributes and contents.""" + + def _invert(h): + "Cheap function to invert a hash." + i = {} + for k,v in h.items(): + i[v] = k + return i + + XML_ENTITIES_TO_SPECIAL_CHARS = { "apos" : "'", + "quot" : '"', + "amp" : "&", + "lt" : "<", + "gt" : ">" } + + XML_SPECIAL_CHARS_TO_ENTITIES = _invert(XML_ENTITIES_TO_SPECIAL_CHARS) + + def _convertEntities(self, match): + """Used in a call to re.sub to replace HTML, XML, and numeric + entities with the appropriate Unicode characters. If HTML + entities are being converted, any unrecognized entities are + escaped.""" + x = match.group(1) + if self.convertHTMLEntities and x in name2codepoint: + return unichr(name2codepoint[x]) + elif x in self.XML_ENTITIES_TO_SPECIAL_CHARS: + if self.convertXMLEntities: + return self.XML_ENTITIES_TO_SPECIAL_CHARS[x] + else: + return u'&%s;' % x + elif len(x) > 0 and x[0] == '#': + # Handle numeric entities + if len(x) > 1 and x[1] == 'x': + return unichr(int(x[2:], 16)) + else: + return unichr(int(x[1:])) + + elif self.escapeUnrecognizedEntities: + return u'&%s;' % x + else: + return u'&%s;' % x + + def __init__(self, parser, name, attrs=None, parent=None, + previous=None): + "Basic constructor." + + # We don't actually store the parser object: that lets extracted + # chunks be garbage-collected + self.parserClass = parser.__class__ + self.isSelfClosing = parser.isSelfClosingTag(name) + self.name = name + if attrs == None: + attrs = [] + self.attrs = attrs + self.contents = [] + self.setup(parent, previous) + self.hidden = False + self.containsSubstitutions = False + self.convertHTMLEntities = parser.convertHTMLEntities + self.convertXMLEntities = parser.convertXMLEntities + self.escapeUnrecognizedEntities = parser.escapeUnrecognizedEntities + + # Convert any HTML, XML, or numeric entities in the attribute values. + convert = lambda(k, val): (k, + re.sub("&(#\d+|#x[0-9a-fA-F]+|\w+);", + self._convertEntities, + val)) + self.attrs = map(convert, self.attrs) + + def get(self, key, default=None): + """Returns the value of the 'key' attribute for the tag, or + the value given for 'default' if it doesn't have that + attribute.""" + return self._getAttrMap().get(key, default) + + def has_key(self, key): + return self._getAttrMap().has_key(key) + + def __getitem__(self, key): + """tag[key] returns the value of the 'key' attribute for the tag, + and throws an exception if it's not there.""" + return self._getAttrMap()[key] + + def __iter__(self): + "Iterating over a tag iterates over its contents." + return iter(self.contents) + + def __len__(self): + "The length of a tag is the length of its list of contents." + return len(self.contents) + + def __contains__(self, x): + return x in self.contents + + def __nonzero__(self): + "A tag is non-None even if it has no contents." + return True + + def __setitem__(self, key, value): + """Setting tag[key] sets the value of the 'key' attribute for the + tag.""" + self._getAttrMap() + self.attrMap[key] = value + found = False + for i in range(0, len(self.attrs)): + if self.attrs[i][0] == key: + self.attrs[i] = (key, value) + found = True + if not found: + self.attrs.append((key, value)) + self._getAttrMap()[key] = value + + def __delitem__(self, key): + "Deleting tag[key] deletes all 'key' attributes for the tag." + for item in self.attrs: + if item[0] == key: + self.attrs.remove(item) + #We don't break because bad HTML can define the same + #attribute multiple times. + self._getAttrMap() + if self.attrMap.has_key(key): + del self.attrMap[key] + + def __call__(self, *args, **kwargs): + """Calling a tag like a function is the same as calling its + findAll() method. Eg. tag('a') returns a list of all the A tags + found within this tag.""" + return apply(self.findAll, args, kwargs) + + def __getattr__(self, tag): + #print "Getattr %s.%s" % (self.__class__, tag) + if len(tag) > 3 and tag.rfind('Tag') == len(tag)-3: + return self.find(tag[:-3]) + elif tag.find('__') != 0: + return self.find(tag) + raise AttributeError, "'%s' object has no attribute '%s'" % (self.__class__, tag) + + def __eq__(self, other): + """Returns true iff this tag has the same name, the same attributes, + and the same contents (recursively) as the given tag. + + NOTE: right now this will return false if two tags have the + same attributes in a different order. Should this be fixed?""" + if not hasattr(other, 'name') or not hasattr(other, 'attrs') or not hasattr(other, 'contents') or self.name != other.name or self.attrs != other.attrs or len(self) != len(other): + return False + for i in range(0, len(self.contents)): + if self.contents[i] != other.contents[i]: + return False + return True + + def __ne__(self, other): + """Returns true iff this tag is not identical to the other tag, + as defined in __eq__.""" + return not self == other + + def __repr__(self, encoding=DEFAULT_OUTPUT_ENCODING): + """Renders this tag as a string.""" + return self.__str__(encoding) + + def __unicode__(self): + return self.__str__(None) + + BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|" + + "&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)" + + ")") + + def _sub_entity(self, x): + """Used with a regular expression to substitute the + appropriate XML entity for an XML special character.""" + return "&" + self.XML_SPECIAL_CHARS_TO_ENTITIES[x.group(0)[0]] + ";" + + def __str__(self, encoding=DEFAULT_OUTPUT_ENCODING, + prettyPrint=False, indentLevel=0): + """Returns a string or Unicode representation of this tag and + its contents. To get Unicode, pass None for encoding. + + NOTE: since Python's HTML parser consumes whitespace, this + method is not certain to reproduce the whitespace present in + the original string.""" + + encodedName = self.toEncoding(self.name, encoding) + + attrs = [] + if self.attrs: + for key, val in self.attrs: + fmt = '%s="%s"' + if isString(val): + if self.containsSubstitutions and '%SOUP-ENCODING%' in val: + val = self.substituteEncoding(val, encoding) + + # The attribute value either: + # + # * Contains no embedded double quotes or single quotes. + # No problem: we enclose it in double quotes. + # * Contains embedded single quotes. No problem: + # double quotes work here too. + # * Contains embedded double quotes. No problem: + # we enclose it in single quotes. + # * Embeds both single _and_ double quotes. This + # can't happen naturally, but it can happen if + # you modify an attribute value after parsing + # the document. Now we have a bit of a + # problem. We solve it by enclosing the + # attribute in single quotes, and escaping any + # embedded single quotes to XML entities. + if '"' in val: + fmt = "%s='%s'" + if "'" in val: + # TODO: replace with apos when + # appropriate. + val = val.replace("'", "&squot;") + + # Now we're okay w/r/t quotes. But the attribute + # value might also contain angle brackets, or + # ampersands that aren't part of entities. We need + # to escape those to XML entities too. + val = self.BARE_AMPERSAND_OR_BRACKET.sub(self._sub_entity, val) + + attrs.append(fmt % (self.toEncoding(key, encoding), + self.toEncoding(val, encoding))) + close = '' + closeTag = '' + if self.isSelfClosing: + close = ' /' + else: + closeTag = '' % encodedName + + indentTag, indentContents = 0, 0 + if prettyPrint: + indentTag = indentLevel + space = (' ' * (indentTag-1)) + indentContents = indentTag + 1 + contents = self.renderContents(encoding, prettyPrint, indentContents) + if self.hidden: + s = contents + else: + s = [] + attributeString = '' + if attrs: + attributeString = ' ' + ' '.join(attrs) + if prettyPrint: + s.append(space) + s.append('<%s%s%s>' % (encodedName, attributeString, close)) + if prettyPrint: + s.append("\n") + s.append(contents) + if prettyPrint and contents and contents[-1] != "\n": + s.append("\n") + if prettyPrint and closeTag: + s.append(space) + s.append(closeTag) + if prettyPrint and closeTag and self.nextSibling: + s.append("\n") + s = ''.join(s) + return s + + def decompose(self): + """Recursively destroys the contents of this tree.""" + contents = [i for i in self.contents] + for i in contents: + if isinstance(i, Tag): + i.decompose() + else: + i.extract() + self.extract() + + def prettify(self, encoding=DEFAULT_OUTPUT_ENCODING): + return self.__str__(encoding, True) + + def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING, + prettyPrint=False, indentLevel=0): + """Renders the contents of this tag as a string in the given + encoding. If encoding is None, returns a Unicode string..""" + s=[] + for c in self: + text = None + if isinstance(c, NavigableString): + text = c.__str__(encoding) + elif isinstance(c, Tag): + s.append(c.__str__(encoding, prettyPrint, indentLevel)) + if text and prettyPrint: + text = text.strip() + if text: + if prettyPrint: + s.append(" " * (indentLevel-1)) + s.append(text) + if prettyPrint: + s.append("\n") + return ''.join(s) + + #Soup methods + + def find(self, name=None, attrs={}, recursive=True, text=None, + **kwargs): + """Return only the first child of this Tag matching the given + criteria.""" + r = None + l = self.findAll(name, attrs, recursive, text, 1, **kwargs) + if l: + r = l[0] + return r + findChild = find + + def findAll(self, name=None, attrs={}, recursive=True, text=None, + limit=None, **kwargs): + """Extracts a list of Tag objects that match the given + criteria. You can specify the name of the Tag and any + attributes you want the Tag to have. + + The value of a key-value pair in the 'attrs' map can be a + string, a list of strings, a regular expression object, or a + callable that takes a string and returns whether or not the + string matches for some custom definition of 'matches'. The + same is true of the tag name.""" + generator = self.recursiveChildGenerator + if not recursive: + generator = self.childGenerator + return self._findAll(name, attrs, text, limit, generator, **kwargs) + findChildren = findAll + + # Pre-3.x compatibility methods + first = find + fetch = findAll + + def fetchText(self, text=None, recursive=True, limit=None): + return self.findAll(text=text, recursive=recursive, limit=limit) + + def firstText(self, text=None, recursive=True): + return self.find(text=text, recursive=recursive) + + #Private methods + + def _getAttrMap(self): + """Initializes a map representation of this tag's attributes, + if not already initialized.""" + if not getattr(self, 'attrMap'): + self.attrMap = {} + for (key, value) in self.attrs: + self.attrMap[key] = value + return self.attrMap + + #Generator methods + def childGenerator(self): + for i in range(0, len(self.contents)): + yield self.contents[i] + raise StopIteration + + def recursiveChildGenerator(self): + stack = [(self, 0)] + while stack: + tag, start = stack.pop() + if isinstance(tag, Tag): + for i in range(start, len(tag.contents)): + a = tag.contents[i] + yield a + if isinstance(a, Tag) and tag.contents: + if i < len(tag.contents) - 1: + stack.append((tag, i+1)) + stack.append((a, 0)) + break + raise StopIteration + +# Next, a couple classes to represent queries and their results. +class SoupStrainer: + """Encapsulates a number of ways of matching a markup element (tag or + text).""" + + def __init__(self, name=None, attrs={}, text=None, **kwargs): + self.name = name + if isString(attrs): + kwargs['class'] = attrs + attrs = None + if kwargs: + if attrs: + attrs = attrs.copy() + attrs.update(kwargs) + else: + attrs = kwargs + self.attrs = attrs + self.text = text + + def __str__(self): + if self.text: + return self.text + else: + return "%s|%s" % (self.name, self.attrs) + + def searchTag(self, markupName=None, markupAttrs={}): + found = None + markup = None + if isinstance(markupName, Tag): + markup = markupName + markupAttrs = markup + callFunctionWithTagData = callable(self.name) \ + and not isinstance(markupName, Tag) + + if (not self.name) \ + or callFunctionWithTagData \ + or (markup and self._matches(markup, self.name)) \ + or (not markup and self._matches(markupName, self.name)): + if callFunctionWithTagData: + match = self.name(markupName, markupAttrs) + else: + match = True + markupAttrMap = None + for attr, matchAgainst in self.attrs.items(): + if not markupAttrMap: + if hasattr(markupAttrs, 'get'): + markupAttrMap = markupAttrs + else: + markupAttrMap = {} + for k,v in markupAttrs: + markupAttrMap[k] = v + attrValue = markupAttrMap.get(attr) + if not self._matches(attrValue, matchAgainst): + match = False + break + if match: + if markup: + found = markup + else: + found = markupName + return found + + def search(self, markup): + #print 'looking for %s in %s' % (self, markup) + found = None + # If given a list of items, scan it for a text element that + # matches. + if isList(markup) and not isinstance(markup, Tag): + for element in markup: + if isinstance(element, NavigableString) \ + and self.search(element): + found = element + break + # If it's a Tag, make sure its name or attributes match. + # Don't bother with Tags if we're searching for text. + elif isinstance(markup, Tag): + if not self.text: + found = self.searchTag(markup) + # If it's text, make sure the text matches. + elif isinstance(markup, NavigableString) or \ + isString(markup): + if self._matches(markup, self.text): + found = markup + else: + raise Exception, "I don't know how to match against a %s" \ + % markup.__class__ + return found + + def _matches(self, markup, matchAgainst): + #print "Matching %s against %s" % (markup, matchAgainst) + result = False + if matchAgainst == True and type(matchAgainst) == types.BooleanType: + result = markup != None + elif callable(matchAgainst): + result = matchAgainst(markup) + else: + #Custom match methods take the tag as an argument, but all + #other ways of matching match the tag name as a string. + if isinstance(markup, Tag): + markup = markup.name + if markup and not isString(markup): + markup = unicode(markup) + #Now we know that chunk is either a string, or None. + if hasattr(matchAgainst, 'match'): + # It's a regexp object. + result = markup and matchAgainst.search(markup) + elif isList(matchAgainst): + result = markup in matchAgainst + elif hasattr(matchAgainst, 'items'): + result = markup.has_key(matchAgainst) + elif matchAgainst and isString(markup): + if isinstance(markup, unicode): + matchAgainst = unicode(matchAgainst) + else: + matchAgainst = str(matchAgainst) + + if not result: + result = matchAgainst == markup + return result + +class ResultSet(list): + """A ResultSet is just a list that keeps track of the SoupStrainer + that created it.""" + def __init__(self, source): + list.__init__([]) + self.source = source + +# Now, some helper functions. + +def isList(l): + """Convenience method that works with all 2.x versions of Python + to determine whether or not something is listlike.""" + return hasattr(l, '__iter__') \ + or (type(l) in (types.ListType, types.TupleType)) + +def isString(s): + """Convenience method that works with all 2.x versions of Python + to determine whether or not something is stringlike.""" + try: + return isinstance(s, unicode) or isinstance(s, basestring) + except NameError: + return isinstance(s, str) + +def buildTagMap(default, *args): + """Turns a list of maps, lists, or scalars into a single map. + Used to build the SELF_CLOSING_TAGS, NESTABLE_TAGS, and + NESTING_RESET_TAGS maps out of lists and partial maps.""" + built = {} + for portion in args: + if hasattr(portion, 'items'): + #It's a map. Merge it. + for k,v in portion.items(): + built[k] = v + elif isList(portion): + #It's a list. Map each item to the default. + for k in portion: + built[k] = default + else: + #It's a scalar. Map it to the default. + built[portion] = default + return built + +# Now, the parser classes. + +class BeautifulStoneSoup(Tag, SGMLParser): + + """This class contains the basic parser and search code. It defines + a parser that knows nothing about tag behavior except for the + following: + + You can't close a tag without closing all the tags it encloses. + That is, "" actually means + "". + + [Another possible explanation is "", but since + this class defines no SELF_CLOSING_TAGS, it will never use that + explanation.] + + This class is useful for parsing XML or made-up markup languages, + or when BeautifulSoup makes an assumption counter to what you were + expecting.""" + + SELF_CLOSING_TAGS = {} + NESTABLE_TAGS = {} + RESET_NESTING_TAGS = {} + QUOTE_TAGS = {} + + MARKUP_MASSAGE = [(re.compile('(<[^<>]*)/>'), + lambda x: x.group(1) + ' />'), + (re.compile(']*)>'), + lambda x: '') + ] + + ROOT_TAG_NAME = u'[document]' + + HTML_ENTITIES = "html" + XML_ENTITIES = "xml" + XHTML_ENTITIES = "xhtml" + # TODO: This only exists for backwards-compatibility + ALL_ENTITIES = XHTML_ENTITIES + + # Used when determining whether a text node is all whitespace and + # can be replaced with a single space. A text node that contains + # fancy Unicode spaces (usually non-breaking) should be left + # alone. + STRIP_ASCII_SPACES = { 9: None, 10: None, 12: None, 13: None, 32: None, } + + def __init__(self, markup="", parseOnlyThese=None, fromEncoding=None, + markupMassage=True, smartQuotesTo=XML_ENTITIES, + convertEntities=None, selfClosingTags=None): + """The Soup object is initialized as the 'root tag', and the + provided markup (which can be a string or a file-like object) + is fed into the underlying parser. + + sgmllib will process most bad HTML, and the BeautifulSoup + class has some tricks for dealing with some HTML that kills + sgmllib, but Beautiful Soup can nonetheless choke or lose data + if your data uses self-closing tags or declarations + incorrectly. + + By default, Beautiful Soup uses regexes to sanitize input, + avoiding the vast majority of these problems. If the problems + don't apply to you, pass in False for markupMassage, and + you'll get better performance. + + The default parser massage techniques fix the two most common + instances of invalid HTML that choke sgmllib: + +
    (No space between name of closing tag and tag close) + (Extraneous whitespace in declaration) + + You can pass in a custom list of (RE object, replace method) + tuples to get Beautiful Soup to scrub your input the way you + want.""" + + self.parseOnlyThese = parseOnlyThese + self.fromEncoding = fromEncoding + self.smartQuotesTo = smartQuotesTo + self.convertEntities = convertEntities + # Set the rules for how we'll deal with the entities we + # encounter + if self.convertEntities: + # It doesn't make sense to convert encoded characters to + # entities even while you're converting entities to Unicode. + # Just convert it all to Unicode. + self.smartQuotesTo = None + if convertEntities == self.HTML_ENTITIES: + self.convertXMLEntities = False + self.convertHTMLEntities = True + self.escapeUnrecognizedEntities = True + elif convertEntities == self.XHTML_ENTITIES: + self.convertXMLEntities = True + self.convertHTMLEntities = True + self.escapeUnrecognizedEntities = False + elif convertEntities == self.XML_ENTITIES: + self.convertXMLEntities = True + self.convertHTMLEntities = False + self.escapeUnrecognizedEntities = False + else: + self.convertXMLEntities = False + self.convertHTMLEntities = False + self.escapeUnrecognizedEntities = False + + self.instanceSelfClosingTags = buildTagMap(None, selfClosingTags) + SGMLParser.__init__(self) + + if hasattr(markup, 'read'): # It's a file-type object. + markup = markup.read() + self.markup = markup + self.markupMassage = markupMassage + try: + self._feed() + except StopParsing: + pass + self.markup = None # The markup can now be GCed + + def convert_charref(self, name): + """This method fixes a bug in Python's SGMLParser.""" + try: + n = int(name) + except ValueError: + return + if not 0 <= n <= 127 : # ASCII ends at 127, not 255 + return + return self.convert_codepoint(n) + + def _feed(self, inDocumentEncoding=None): + # Convert the document to Unicode. + markup = self.markup + if isinstance(markup, unicode): + if not hasattr(self, 'originalEncoding'): + self.originalEncoding = None + else: + dammit = UnicodeDammit\ + (markup, [self.fromEncoding, inDocumentEncoding], + smartQuotesTo=self.smartQuotesTo) + markup = dammit.unicode + self.originalEncoding = dammit.originalEncoding + if markup: + if self.markupMassage: + if not isList(self.markupMassage): + self.markupMassage = self.MARKUP_MASSAGE + for fix, m in self.markupMassage: + markup = fix.sub(m, markup) + # TODO: We get rid of markupMassage so that the + # soup object can be deepcopied later on. Some + # Python installations can't copy regexes. If anyone + # was relying on the existence of markupMassage, this + # might cause problems. + del(self.markupMassage) + self.reset() + + SGMLParser.feed(self, markup) + # Close out any unfinished strings and close all the open tags. + self.endData() + while self.currentTag.name != self.ROOT_TAG_NAME: + self.popTag() + + def __getattr__(self, methodName): + """This method routes method call requests to either the SGMLParser + superclass or the Tag superclass, depending on the method name.""" + #print "__getattr__ called on %s.%s" % (self.__class__, methodName) + + if methodName.find('start_') == 0 or methodName.find('end_') == 0 \ + or methodName.find('do_') == 0: + return SGMLParser.__getattr__(self, methodName) + elif methodName.find('__') != 0: + return Tag.__getattr__(self, methodName) + else: + raise AttributeError + + def isSelfClosingTag(self, name): + """Returns true iff the given string is the name of a + self-closing tag according to this parser.""" + return self.SELF_CLOSING_TAGS.has_key(name) \ + or self.instanceSelfClosingTags.has_key(name) + + def reset(self): + Tag.__init__(self, self, self.ROOT_TAG_NAME) + self.hidden = 1 + SGMLParser.reset(self) + self.currentData = [] + self.currentTag = None + self.tagStack = [] + self.quoteStack = [] + self.pushTag(self) + + def popTag(self): + tag = self.tagStack.pop() + # Tags with just one string-owning child get the child as a + # 'string' property, so that soup.tag.string is shorthand for + # soup.tag.contents[0] + if len(self.currentTag.contents) == 1 and \ + isinstance(self.currentTag.contents[0], NavigableString): + self.currentTag.string = self.currentTag.contents[0] + + #print "Pop", tag.name + if self.tagStack: + self.currentTag = self.tagStack[-1] + return self.currentTag + + def pushTag(self, tag): + #print "Push", tag.name + if self.currentTag: + self.currentTag.contents.append(tag) + self.tagStack.append(tag) + self.currentTag = self.tagStack[-1] + + def endData(self, containerClass=NavigableString): + if self.currentData: + currentData = ''.join(self.currentData) + if not currentData.translate(self.STRIP_ASCII_SPACES): + if '\n' in currentData: + currentData = '\n' + else: + currentData = ' ' + self.currentData = [] + if self.parseOnlyThese and len(self.tagStack) <= 1 and \ + (not self.parseOnlyThese.text or \ + not self.parseOnlyThese.search(currentData)): + return + o = containerClass(currentData) + o.setup(self.currentTag, self.previous) + if self.previous: + self.previous.next = o + self.previous = o + self.currentTag.contents.append(o) + + + def _popToTag(self, name, inclusivePop=True): + """Pops the tag stack up to and including the most recent + instance of the given tag. If inclusivePop is false, pops the tag + stack up to but *not* including the most recent instqance of + the given tag.""" + #print "Popping to %s" % name + if name == self.ROOT_TAG_NAME: + return + + numPops = 0 + mostRecentTag = None + for i in range(len(self.tagStack)-1, 0, -1): + if name == self.tagStack[i].name: + numPops = len(self.tagStack)-i + break + if not inclusivePop: + numPops = numPops - 1 + + for i in range(0, numPops): + mostRecentTag = self.popTag() + return mostRecentTag + + def _smartPop(self, name): + + """We need to pop up to the previous tag of this type, unless + one of this tag's nesting reset triggers comes between this + tag and the previous tag of this type, OR unless this tag is a + generic nesting trigger and another generic nesting trigger + comes between this tag and the previous tag of this type. + + Examples: +

    FooBar *

    * should pop to 'p', not 'b'. +

    FooBar *

    * should pop to 'table', not 'p'. +

    Foo

    Bar *

    * should pop to 'tr', not 'p'. + +

    • *
    • * should pop to 'ul', not the first 'li'. +
    • ** should pop to 'table', not the first 'tr' + tag should + implicitly close the previous tag within the same
    ** should pop to 'tr', not the first 'td' + """ + + nestingResetTriggers = self.NESTABLE_TAGS.get(name) + isNestable = nestingResetTriggers != None + isResetNesting = self.RESET_NESTING_TAGS.has_key(name) + popTo = None + inclusive = True + for i in range(len(self.tagStack)-1, 0, -1): + p = self.tagStack[i] + if (not p or p.name == name) and not isNestable: + #Non-nestable tags get popped to the top or to their + #last occurance. + popTo = name + break + if (nestingResetTriggers != None + and p.name in nestingResetTriggers) \ + or (nestingResetTriggers == None and isResetNesting + and self.RESET_NESTING_TAGS.has_key(p.name)): + + #If we encounter one of the nesting reset triggers + #peculiar to this tag, or we encounter another tag + #that causes nesting to reset, pop up to but not + #including that tag. + popTo = p.name + inclusive = False + break + p = p.parent + if popTo: + self._popToTag(popTo, inclusive) + + def unknown_starttag(self, name, attrs, selfClosing=0): + #print "Start tag %s: %s" % (name, attrs) + if self.quoteStack: + #This is not a real tag. + #print "<%s> is not real!" % name + attrs = ''.join(map(lambda(x, y): ' %s="%s"' % (x, y), attrs)) + self.handle_data('<%s%s>' % (name, attrs)) + return + self.endData() + + if not self.isSelfClosingTag(name) and not selfClosing: + self._smartPop(name) + + if self.parseOnlyThese and len(self.tagStack) <= 1 \ + and (self.parseOnlyThese.text or not self.parseOnlyThese.searchTag(name, attrs)): + return + + tag = Tag(self, name, attrs, self.currentTag, self.previous) + if self.previous: + self.previous.next = tag + self.previous = tag + self.pushTag(tag) + if selfClosing or self.isSelfClosingTag(name): + self.popTag() + if name in self.QUOTE_TAGS: + #print "Beginning quote (%s)" % name + self.quoteStack.append(name) + self.literal = 1 + return tag + + def unknown_endtag(self, name): + #print "End tag %s" % name + if self.quoteStack and self.quoteStack[-1] != name: + #This is not a real end tag. + #print " is not real!" % name + self.handle_data('' % name) + return + self.endData() + self._popToTag(name) + if self.quoteStack and self.quoteStack[-1] == name: + self.quoteStack.pop() + self.literal = (len(self.quoteStack) > 0) + + def handle_data(self, data): + self.currentData.append(data) + + def _toStringSubclass(self, text, subclass): + """Adds a certain piece of text to the tree as a NavigableString + subclass.""" + self.endData() + self.handle_data(text) + self.endData(subclass) + + def handle_pi(self, text): + """Handle a processing instruction as a ProcessingInstruction + object, possibly one with a %SOUP-ENCODING% slot into which an + encoding will be plugged later.""" + if text[:3] == "xml": + text = u"xml version='1.0' encoding='%SOUP-ENCODING%'" + self._toStringSubclass(text, ProcessingInstruction) + + def handle_comment(self, text): + "Handle comments as Comment objects." + self._toStringSubclass(text, Comment) + + def handle_charref(self, ref): + "Handle character references as data." + if self.convertEntities: + data = unichr(int(ref)) + else: + data = '&#%s;' % ref + self.handle_data(data) + + def handle_entityref(self, ref): + """Handle entity references as data, possibly converting known + HTML and/or XML entity references to the corresponding Unicode + characters.""" + data = None + if self.convertHTMLEntities: + try: + data = unichr(name2codepoint[ref]) + except KeyError: + pass + + if not data and self.convertXMLEntities: + data = self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref) + + if not data and self.convertHTMLEntities and \ + not self.XML_ENTITIES_TO_SPECIAL_CHARS.get(ref): + # TODO: We've got a problem here. We're told this is + # an entity reference, but it's not an XML entity + # reference or an HTML entity reference. Nonetheless, + # the logical thing to do is to pass it through as an + # unrecognized entity reference. + # + # Except: when the input is "&carol;" this function + # will be called with input "carol". When the input is + # "AT&T", this function will be called with input + # "T". We have no way of knowing whether a semicolon + # was present originally, so we don't know whether + # this is an unknown entity or just a misplaced + # ampersand. + # + # The more common case is a misplaced ampersand, so I + # escape the ampersand and omit the trailing semicolon. + data = "&%s" % ref + if not data: + # This case is different from the one above, because we + # haven't already gone through a supposedly comprehensive + # mapping of entities to Unicode characters. We might not + # have gone through any mapping at all. So the chances are + # very high that this is a real entity, and not a + # misplaced ampersand. + data = "&%s;" % ref + self.handle_data(data) + + def handle_decl(self, data): + "Handle DOCTYPEs and the like as Declaration objects." + self._toStringSubclass(data, Declaration) + + def parse_declaration(self, i): + """Treat a bogus SGML declaration as raw data. Treat a CDATA + declaration as a CData object.""" + j = None + if self.rawdata[i:i+9] == '', i) + if k == -1: + k = len(self.rawdata) + data = self.rawdata[i+9:k] + j = k+3 + self._toStringSubclass(data, CData) + else: + try: + j = SGMLParser.parse_declaration(self, i) + except SGMLParseError: + toHandle = self.rawdata[i:] + self.handle_data(toHandle) + j = i + len(toHandle) + return j + +class BeautifulSoup(BeautifulStoneSoup): + + """This parser knows the following facts about HTML: + + * Some tags have no closing tag and should be interpreted as being + closed as soon as they are encountered. + + * The text inside some tags (ie. 'script') may contain tags which + are not really part of the document and which should be parsed + as text, not tags. If you want to parse the text as tags, you can + always fetch it and parse it explicitly. + + * Tag nesting rules: + + Most tags can't be nested at all. For instance, the occurance of + a

    tag should implicitly close the previous

    tag. + +

    Para1

    Para2 + should be transformed into: +

    Para1

    Para2 + + Some tags can be nested arbitrarily. For instance, the occurance + of a

    tag should _not_ implicitly close the previous +
    tag. + + Alice said:
    Bob said:
    Blah + should NOT be transformed into: + Alice said:
    Bob said:
    Blah + + Some tags can be nested, but the nesting is reset by the + interposition of other tags. For instance, a
    , + but not close a tag in another table. + +
    BlahBlah + should be transformed into: +
    BlahBlah + but, + Blah
    Blah + should NOT be transformed into + Blah
    Blah + + Differing assumptions about tag nesting rules are a major source + of problems with the BeautifulSoup class. If BeautifulSoup is not + treating as nestable a tag your page author treats as nestable, + try ICantBelieveItsBeautifulSoup, MinimalSoup, or + BeautifulStoneSoup before writing your own subclass.""" + + def __init__(self, *args, **kwargs): + if not kwargs.has_key('smartQuotesTo'): + kwargs['smartQuotesTo'] = self.HTML_ENTITIES + BeautifulStoneSoup.__init__(self, *args, **kwargs) + + SELF_CLOSING_TAGS = buildTagMap(None, + ['br' , 'hr', 'input', 'img', 'meta', + 'spacer', 'link', 'frame', 'base']) + + QUOTE_TAGS = {'script' : None, 'textarea' : None} + + #According to the HTML standard, each of these inline tags can + #contain another tag of the same type. Furthermore, it's common + #to actually use these tags this way. + NESTABLE_INLINE_TAGS = ['span', 'font', 'q', 'object', 'bdo', 'sub', 'sup', + 'center'] + + #According to the HTML standard, these block tags can contain + #another tag of the same type. Furthermore, it's common + #to actually use these tags this way. + NESTABLE_BLOCK_TAGS = ['blockquote', 'div', 'fieldset', 'ins', 'del'] + + #Lists can contain other lists, but there are restrictions. + NESTABLE_LIST_TAGS = { 'ol' : [], + 'ul' : [], + 'li' : ['ul', 'ol'], + 'dl' : [], + 'dd' : ['dl'], + 'dt' : ['dl'] } + + #Tables can contain other tables, but there are restrictions. + NESTABLE_TABLE_TAGS = {'table' : [], + 'tr' : ['table', 'tbody', 'tfoot', 'thead'], + 'td' : ['tr'], + 'th' : ['tr'], + 'thead' : ['table'], + 'tbody' : ['table'], + 'tfoot' : ['table'], + } + + NON_NESTABLE_BLOCK_TAGS = ['address', 'form', 'p', 'pre'] + + #If one of these tags is encountered, all tags up to the next tag of + #this type are popped. + RESET_NESTING_TAGS = buildTagMap(None, NESTABLE_BLOCK_TAGS, 'noscript', + NON_NESTABLE_BLOCK_TAGS, + NESTABLE_LIST_TAGS, + NESTABLE_TABLE_TAGS) + + NESTABLE_TAGS = buildTagMap([], NESTABLE_INLINE_TAGS, NESTABLE_BLOCK_TAGS, + NESTABLE_LIST_TAGS, NESTABLE_TABLE_TAGS) + + # Used to detect the charset in a META tag; see start_meta + CHARSET_RE = re.compile("((^|;)\s*charset=)([^;]*)") + + def start_meta(self, attrs): + """Beautiful Soup can detect a charset included in a META tag, + try to convert the document to that charset, and re-parse the + document from the beginning.""" + httpEquiv = None + contentType = None + contentTypeIndex = None + tagNeedsEncodingSubstitution = False + + for i in range(0, len(attrs)): + key, value = attrs[i] + key = key.lower() + if key == 'http-equiv': + httpEquiv = value + elif key == 'content': + contentType = value + contentTypeIndex = i + + if httpEquiv and contentType: # It's an interesting meta tag. + match = self.CHARSET_RE.search(contentType) + if match: + if getattr(self, 'declaredHTMLEncoding') or \ + (self.originalEncoding == self.fromEncoding): + # This is our second pass through the document, or + # else an encoding was specified explicitly and it + # worked. Rewrite the meta tag. + newAttr = self.CHARSET_RE.sub\ + (lambda(match):match.group(1) + + "%SOUP-ENCODING%", contentType) + attrs[contentTypeIndex] = (attrs[contentTypeIndex][0], + newAttr) + tagNeedsEncodingSubstitution = True + else: + # This is our first pass through the document. + # Go through it again with the new information. + newCharset = match.group(3) + if newCharset and newCharset != self.originalEncoding: + self.declaredHTMLEncoding = newCharset + self._feed(self.declaredHTMLEncoding) + raise StopParsing + tag = self.unknown_starttag("meta", attrs) + if tag and tagNeedsEncodingSubstitution: + tag.containsSubstitutions = True + +class StopParsing(Exception): + pass + +class ICantBelieveItsBeautifulSoup(BeautifulSoup): + + """The BeautifulSoup class is oriented towards skipping over + common HTML errors like unclosed tags. However, sometimes it makes + errors of its own. For instance, consider this fragment: + + FooBar + + This is perfectly valid (if bizarre) HTML. However, the + BeautifulSoup class will implicitly close the first b tag when it + encounters the second 'b'. It will think the author wrote + "FooBar", and didn't close the first 'b' tag, because + there's no real-world reason to bold something that's already + bold. When it encounters '' it will close two more 'b' + tags, for a grand total of three tags closed instead of two. This + can throw off the rest of your document structure. The same is + true of a number of other tags, listed below. + + It's much more common for someone to forget to close a 'b' tag + than to actually use nested 'b' tags, and the BeautifulSoup class + handles the common case. This class handles the not-co-common + case: where you can't believe someone wrote what they did, but + it's valid HTML and BeautifulSoup screwed up by assuming it + wouldn't be.""" + + I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS = \ + ['em', 'big', 'i', 'small', 'tt', 'abbr', 'acronym', 'strong', + 'cite', 'code', 'dfn', 'kbd', 'samp', 'strong', 'var', 'b', + 'big'] + + I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS = ['noscript'] + + NESTABLE_TAGS = buildTagMap([], BeautifulSoup.NESTABLE_TAGS, + I_CANT_BELIEVE_THEYRE_NESTABLE_BLOCK_TAGS, + I_CANT_BELIEVE_THEYRE_NESTABLE_INLINE_TAGS) + +class MinimalSoup(BeautifulSoup): + """The MinimalSoup class is for parsing HTML that contains + pathologically bad markup. It makes no assumptions about tag + nesting, but it does know which tags are self-closing, that + attack + '/': '\\/', + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +# assume this produces an infinity on all machines (probably not guaranteed) +INFINITY = float('1e66666') + +def floatstr(o, allow_nan=True): + # Check for specials. Note that this type of test is processor- and/or + # platform-specific, so do tests which don't depend on the internals. + + if o != o: + text = 'NaN' + elif o == INFINITY: + text = 'Infinity' + elif o == -INFINITY: + text = '-Infinity' + else: + return str(o) + + if not allow_nan: + raise ValueError("Out of range float values are not JSON compliant: %r" + % (o,)) + + return text + + +def encode_basestring(s): + """ + Return a JSON representation of a Python string + """ + def replace(match): + return ESCAPE_DCT[match.group(0)] + return '"' + ESCAPE.sub(replace, s) + '"' + +def encode_basestring_ascii(s): + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + +try: + encode_basestring_ascii = _speedups.encode_basestring_ascii + _need_utf8 = True +except AttributeError: + _need_utf8 = False + +class JSONEncoder(object): + """ + Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + """ + __all__ = ['__init__', 'default', 'encode', 'iterencode'] + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8'): + """ + Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is False, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is True, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is True, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is True, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is True, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a non-negative integer, then JSON array + elements and object members will be pretty-printed with that + indent level. An indent level of 0 will only insert newlines. + None is the most compact representation. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.indent = indent + self.current_indent_level = 0 + if separators is not None: + self.item_separator, self.key_separator = separators + self.encoding = encoding + + def _newline_indent(self): + return '\n' + (' ' * (self.indent * self.current_indent_level)) + + def _iterencode_list(self, lst, markers=None): + if not lst: + yield '[]' + return + if markers is not None: + markerid = id(lst) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = lst + yield '[' + if self.indent is not None: + self.current_indent_level += 1 + newline_indent = self._newline_indent() + separator = self.item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + separator = self.item_separator + first = True + for value in lst: + if first: + first = False + else: + yield separator + for chunk in self._iterencode(value, markers): + yield chunk + if newline_indent is not None: + self.current_indent_level -= 1 + yield self._newline_indent() + yield ']' + if markers is not None: + del markers[markerid] + + def _iterencode_dict(self, dct, markers=None): + if not dct: + yield '{}' + return + if markers is not None: + markerid = id(dct) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = dct + yield '{' + key_separator = self.key_separator + if self.indent is not None: + self.current_indent_level += 1 + newline_indent = self._newline_indent() + item_separator = self.item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + item_separator = self.item_separator + first = True + if self.ensure_ascii: + encoder = encode_basestring_ascii + else: + encoder = encode_basestring + allow_nan = self.allow_nan + if self.sort_keys: + keys = dct.keys() + keys.sort() + items = [(k, dct[k]) for k in keys] + else: + items = dct.iteritems() + _encoding = self.encoding + _do_decode = (_encoding is not None + and not (_need_utf8 and _encoding == 'utf-8')) + for key, value in items: + if isinstance(key, str): + if _do_decode: + key = key.decode(_encoding) + elif isinstance(key, basestring): + pass + # JavaScript is weakly typed for these, so it makes sense to + # also allow them. Many encoders seem to do something like this. + elif isinstance(key, float): + key = floatstr(key, allow_nan) + elif isinstance(key, (int, long)): + key = str(key) + elif key is True: + key = 'true' + elif key is False: + key = 'false' + elif key is None: + key = 'null' + elif self.skipkeys: + continue + else: + raise TypeError("key %r is not a string" % (key,)) + if first: + first = False + else: + yield item_separator + yield encoder(key) + yield key_separator + for chunk in self._iterencode(value, markers): + yield chunk + if newline_indent is not None: + self.current_indent_level -= 1 + yield self._newline_indent() + yield '}' + if markers is not None: + del markers[markerid] + + def _iterencode(self, o, markers=None): + if isinstance(o, basestring): + if self.ensure_ascii: + encoder = encode_basestring_ascii + else: + encoder = encode_basestring + _encoding = self.encoding + if (_encoding is not None and isinstance(o, str) + and not (_need_utf8 and _encoding == 'utf-8')): + o = o.decode(_encoding) + yield encoder(o) + elif o is None: + yield 'null' + elif o is True: + yield 'true' + elif o is False: + yield 'false' + elif isinstance(o, (int, long)): + yield str(o) + elif isinstance(o, float): + yield floatstr(o, self.allow_nan) + elif isinstance(o, (list, tuple)): + for chunk in self._iterencode_list(o, markers): + yield chunk + elif isinstance(o, dict): + for chunk in self._iterencode_dict(o, markers): + yield chunk + else: + if markers is not None: + markerid = id(o) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = o + for chunk in self._iterencode_default(o, markers): + yield chunk + if markers is not None: + del markers[markerid] + + def _iterencode_default(self, o, markers=None): + newobj = self.default(o) + return self._iterencode(newobj, markers) + + def default(self, o): + """ + Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + """ + raise TypeError("%r is not JSON serializable" % (o,)) + + def encode(self, o): + """ + Return a JSON string representation of a Python data structure. + + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo":["bar", "baz"]}' + """ + # This is for extremely simple cases and benchmarks... + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8' and _need_utf8)): + o = o.decode(_encoding) + return encode_basestring_ascii(o) + # This doesn't pass the iterator directly to ''.join() because it + # sucks at reporting exceptions. It's going to do this internally + # anyway because it uses PySequence_Fast or similar. + chunks = list(self.iterencode(o)) + return ''.join(chunks) + + def iterencode(self, o): + """ + Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + """ + if self.check_circular: + markers = {} + else: + markers = None + return self._iterencode(o, markers) + +__all__ = ['JSONEncoder'] diff --git a/scrapy/trunk/scrapy/lib/simplejson/jsonfilter.py b/scrapy/trunk/scrapy/lib/simplejson/jsonfilter.py new file mode 100644 index 000000000..01ca21df6 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/jsonfilter.py @@ -0,0 +1,40 @@ +import simplejson +import cgi + +class JSONFilter(object): + def __init__(self, app, mime_type='text/x-json'): + self.app = app + self.mime_type = mime_type + + def __call__(self, environ, start_response): + # Read JSON POST input to jsonfilter.json if matching mime type + response = {'status': '200 OK', 'headers': []} + def json_start_response(status, headers): + response['status'] = status + response['headers'].extend(headers) + environ['jsonfilter.mime_type'] = self.mime_type + if environ.get('REQUEST_METHOD', '') == 'POST': + if environ.get('CONTENT_TYPE', '') == self.mime_type: + args = [_ for _ in [environ.get('CONTENT_LENGTH')] if _] + data = environ['wsgi.input'].read(*map(int, args)) + environ['jsonfilter.json'] = simplejson.loads(data) + res = simplejson.dumps(self.app(environ, json_start_response)) + jsonp = cgi.parse_qs(environ.get('QUERY_STRING', '')).get('jsonp') + if jsonp: + content_type = 'text/javascript' + res = ''.join(jsonp + ['(', res, ')']) + elif 'Opera' in environ.get('HTTP_USER_AGENT', ''): + # Opera has bunk XMLHttpRequest support for most mime types + content_type = 'text/plain' + else: + content_type = self.mime_type + headers = [ + ('Content-type', content_type), + ('Content-length', len(res)), + ] + headers.extend(response['headers']) + start_response(response['status'], headers) + return [res] + +def factory(app, global_conf, **kw): + return JSONFilter(app, **kw) diff --git a/scrapy/trunk/scrapy/lib/simplejson/scanner.py b/scrapy/trunk/scrapy/lib/simplejson/scanner.py new file mode 100644 index 000000000..64f4999fb --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/scanner.py @@ -0,0 +1,63 @@ +""" +Iterator based sre token scanner +""" +import sre_parse, sre_compile, sre_constants +from sre_constants import BRANCH, SUBPATTERN +from re import VERBOSE, MULTILINE, DOTALL +import re + +__all__ = ['Scanner', 'pattern'] + +FLAGS = (VERBOSE | MULTILINE | DOTALL) +class Scanner(object): + def __init__(self, lexicon, flags=FLAGS): + self.actions = [None] + # combine phrases into a compound pattern + s = sre_parse.Pattern() + s.flags = flags + p = [] + for idx, token in enumerate(lexicon): + phrase = token.pattern + try: + subpattern = sre_parse.SubPattern(s, + [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) + except sre_constants.error: + raise + p.append(subpattern) + self.actions.append(token) + + p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) + self.scanner = sre_compile.compile(p) + + + def iterscan(self, string, idx=0, context=None): + """ + Yield match, end_idx for each match + """ + match = self.scanner.scanner(string, idx).match + actions = self.actions + lastend = idx + end = len(string) + while True: + m = match() + if m is None: + break + matchbegin, matchend = m.span() + if lastend == matchend: + break + action = actions[m.lastindex] + if action is not None: + rval, next_pos = action(m, context) + if next_pos is not None and next_pos != matchend: + # "fast forward" the scanner + matchend = next_pos + match = self.scanner.scanner(string, matchend).match + yield rval, matchend + lastend = matchend + +def pattern(pattern, flags=FLAGS): + def decorator(fn): + fn.pattern = pattern + fn.regex = re.compile(pattern, flags) + return fn + return decorator diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/__init__.py b/scrapy/trunk/scrapy/lib/simplejson/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_attacks.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_attacks.py new file mode 100644 index 000000000..8ecfed8f2 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_attacks.py @@ -0,0 +1,6 @@ +def test_script_close_attack(): + import simplejson + res = simplejson.dumps('') + assert '' not in res + res = simplejson.dumps(simplejson.loads('""')) + assert '' not in res diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_dump.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_dump.py new file mode 100644 index 000000000..b4e236e56 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_dump.py @@ -0,0 +1,10 @@ +from cStringIO import StringIO +import simplejson as S + +def test_dump(): + sio = StringIO() + S.dump({}, sio) + assert sio.getvalue() == '{}' + +def test_dumps(): + assert S.dumps({}) == '{}' diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_fail.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_fail.py new file mode 100644 index 000000000..a99d9c40c --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_fail.py @@ -0,0 +1,70 @@ +# Fri Dec 30 18:57:26 2005 +JSONDOCS = [ + # http://json.org/JSON_checker/test/fail1.json + '"A JSON payload should be an object or array, not a string."', + # http://json.org/JSON_checker/test/fail2.json + '["Unclosed array"', + # http://json.org/JSON_checker/test/fail3.json + '{unquoted_key: "keys must be quoted}', + # http://json.org/JSON_checker/test/fail4.json + '["extra comma",]', + # http://json.org/JSON_checker/test/fail5.json + '["double extra comma",,]', + # http://json.org/JSON_checker/test/fail6.json + '[ , "<-- missing value"]', + # http://json.org/JSON_checker/test/fail7.json + '["Comma after the close"],', + # http://json.org/JSON_checker/test/fail8.json + '["Extra close"]]', + # http://json.org/JSON_checker/test/fail9.json + '{"Extra comma": true,}', + # http://json.org/JSON_checker/test/fail10.json + '{"Extra value after close": true} "misplaced quoted value"', + # http://json.org/JSON_checker/test/fail11.json + '{"Illegal expression": 1 + 2}', + # http://json.org/JSON_checker/test/fail12.json + '{"Illegal invocation": alert()}', + # http://json.org/JSON_checker/test/fail13.json + '{"Numbers cannot have leading zeroes": 013}', + # http://json.org/JSON_checker/test/fail14.json + '{"Numbers cannot be hex": 0x14}', + # http://json.org/JSON_checker/test/fail15.json + '["Illegal backslash escape: \\x15"]', + # http://json.org/JSON_checker/test/fail16.json + '["Illegal backslash escape: \\\'"]', + # http://json.org/JSON_checker/test/fail17.json + '["Illegal backslash escape: \\017"]', + # http://json.org/JSON_checker/test/fail18.json + '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', + # http://json.org/JSON_checker/test/fail19.json + '{"Missing colon" null}', + # http://json.org/JSON_checker/test/fail20.json + '{"Double colon":: null}', + # http://json.org/JSON_checker/test/fail21.json + '{"Comma instead of colon", null}', + # http://json.org/JSON_checker/test/fail22.json + '["Colon instead of comma": false]', + # http://json.org/JSON_checker/test/fail23.json + '["Bad value", truth]', + # http://json.org/JSON_checker/test/fail24.json + "['single quote']", +] + +SKIPS = { + 1: "why not have a string payload?", + 18: "spec doesn't specify any nesting limitations", +} + +def test_failures(): + import simplejson + for idx, doc in enumerate(JSONDOCS): + idx = idx + 1 + if idx in SKIPS: + simplejson.loads(doc) + continue + try: + simplejson.loads(doc) + except ValueError: + pass + else: + assert False, "Expected failure for fail%d.json: %r" % (idx, doc) diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_indent.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_indent.py new file mode 100644 index 000000000..47dd4dc2e --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_indent.py @@ -0,0 +1,41 @@ + + + +def test_indent(): + import simplejson + import textwrap + + h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', + {'nifty': 87}, {'field': 'yes', 'morefield': False} ] + + expect = textwrap.dedent("""\ + [ + [ + "blorpie" + ], + [ + "whoops" + ], + [], + "d-shtaeou", + "d-nthiouh", + "i-vhbjkhnth", + { + "nifty": 87 + }, + { + "field": "yes", + "morefield": false + } + ]""") + + + d1 = simplejson.dumps(h) + d2 = simplejson.dumps(h, indent=2, sort_keys=True, separators=(',', ': ')) + + h1 = simplejson.loads(d1) + h2 = simplejson.loads(d2) + + assert h1 == h + assert h2 == h + assert d2 == expect diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass1.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass1.py new file mode 100644 index 000000000..4eda19258 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass1.py @@ -0,0 +1,72 @@ +# from http://json.org/JSON_checker/test/pass1.json +JSON = r''' +[ + "JSON Test Pattern pass1", + {"object with 1 member":["array with 1 element"]}, + {}, + [], + -42, + true, + false, + null, + { + "integer": 1234567890, + "real": -9876.543210, + "e": 0.123456789e-12, + "E": 1.234567890E+34, + "": 23456789012E666, + "zero": 0, + "one": 1, + "space": " ", + "quote": "\"", + "backslash": "\\", + "controls": "\b\f\n\r\t", + "slash": "/ & \/", + "alpha": "abcdefghijklmnopqrstuvwyz", + "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", + "digit": "0123456789", + "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", + "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", + "true": true, + "false": false, + "null": null, + "array":[ ], + "object":{ }, + "address": "50 St. James Street", + "url": "http://www.JSON.org/", + "comment": "// /* */": " ", + " s p a c e d " :[1,2 , 3 + +, + +4 , 5 , 6 ,7 ], + "compact": [1,2,3,4,5,6,7], + "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", + "quotes": "" \u0022 %22 0x22 034 "", + "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" +: "A key can be any string" + }, + 0.5 ,98.6 +, +99.44 +, + +1066 + + +,"rosebud"] +''' + +def test_parse(): + # test in/out equivalence and parsing + import simplejson + res = simplejson.loads(JSON) + out = simplejson.dumps(res) + assert res == simplejson.loads(out) + try: + simplejson.dumps(res, allow_nan=False) + except ValueError: + pass + else: + assert False, "23456789012E666 should be out of range" diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass2.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass2.py new file mode 100644 index 000000000..ae74abbf5 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass2.py @@ -0,0 +1,11 @@ +# from http://json.org/JSON_checker/test/pass2.json +JSON = r''' +[[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] +''' + +def test_parse(): + # test in/out equivalence and parsing + import simplejson + res = simplejson.loads(JSON) + out = simplejson.dumps(res) + assert res == simplejson.loads(out) diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass3.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass3.py new file mode 100644 index 000000000..d94893ffa --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_pass3.py @@ -0,0 +1,16 @@ +# from http://json.org/JSON_checker/test/pass3.json +JSON = r''' +{ + "JSON Test Pattern pass3": { + "The outermost value": "must be an object or array.", + "In this test": "It is an object." + } +} +''' + +def test_parse(): + # test in/out equivalence and parsing + import simplejson + res = simplejson.loads(JSON) + out = simplejson.dumps(res) + assert res == simplejson.loads(out) diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_recursion.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_recursion.py new file mode 100644 index 000000000..756b06612 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_recursion.py @@ -0,0 +1,62 @@ +import simplejson + +def test_listrecursion(): + x = [] + x.append(x) + try: + simplejson.dumps(x) + except ValueError: + pass + else: + assert False, "didn't raise ValueError on list recursion" + x = [] + y = [x] + x.append(y) + try: + simplejson.dumps(x) + except ValueError: + pass + else: + assert False, "didn't raise ValueError on alternating list recursion" + y = [] + x = [y, y] + # ensure that the marker is cleared + simplejson.dumps(x) + +def test_dictrecursion(): + x = {} + x["test"] = x + try: + simplejson.dumps(x) + except ValueError: + pass + else: + assert False, "didn't raise ValueError on dict recursion" + x = {} + y = {"a": x, "b": x} + # ensure that the marker is cleared + simplejson.dumps(x) + +class TestObject: + pass + +class RecursiveJSONEncoder(simplejson.JSONEncoder): + recurse = False + def default(self, o): + if o is TestObject: + if self.recurse: + return [TestObject] + else: + return 'TestObject' + simplejson.JSONEncoder.default(o) + +def test_defaultrecursion(): + enc = RecursiveJSONEncoder() + assert enc.encode(TestObject) == '"TestObject"' + enc.recurse = True + try: + enc.encode(TestObject) + except ValueError: + pass + else: + assert False, "didn't raise ValueError on default recursion" diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_separators.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_separators.py new file mode 100644 index 000000000..a61535478 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_separators.py @@ -0,0 +1,41 @@ + + + +def test_separators(): + import simplejson + import textwrap + + h = [['blorpie'], ['whoops'], [], 'd-shtaeou', 'd-nthiouh', 'i-vhbjkhnth', + {'nifty': 87}, {'field': 'yes', 'morefield': False} ] + + expect = textwrap.dedent("""\ + [ + [ + "blorpie" + ] , + [ + "whoops" + ] , + [] , + "d-shtaeou" , + "d-nthiouh" , + "i-vhbjkhnth" , + { + "nifty" : 87 + } , + { + "field" : "yes" , + "morefield" : false + } + ]""") + + + d1 = simplejson.dumps(h) + d2 = simplejson.dumps(h, indent=2, sort_keys=True, separators=(' ,', ' : ')) + + h1 = simplejson.loads(d1) + h2 = simplejson.loads(d2) + + assert h1 == h + assert h2 == h + assert d2 == expect diff --git a/scrapy/trunk/scrapy/lib/simplejson/tests/test_unicode.py b/scrapy/trunk/scrapy/lib/simplejson/tests/test_unicode.py new file mode 100644 index 000000000..88d09393e --- /dev/null +++ b/scrapy/trunk/scrapy/lib/simplejson/tests/test_unicode.py @@ -0,0 +1,16 @@ +import simplejson as S + +def test_encoding1(): + encoder = S.JSONEncoder(encoding='utf-8') + u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' + s = u.encode('utf-8') + ju = encoder.encode(u) + js = encoder.encode(s) + assert ju == js + +def test_encoding2(): + u = u'\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' + s = u.encode('utf-8') + ju = S.dumps(u, encoding='utf-8') + js = S.dumps(s, encoding='utf-8') + assert ju == js diff --git a/scrapy/trunk/scrapy/lib/spidermonkey/INSTALL.scrapy b/scrapy/trunk/scrapy/lib/spidermonkey/INSTALL.scrapy new file mode 100644 index 000000000..6eef8b6ee --- /dev/null +++ b/scrapy/trunk/scrapy/lib/spidermonkey/INSTALL.scrapy @@ -0,0 +1,27 @@ +Spidermonkey bindings for python (based on standard library ctypes) + +Quick instructions to install with scrapy package. + +* go to http://ftp.mozilla.org/pub/mozilla.org/js/ and download the +release 1.7.0. + +* Extract the source, go to js/src and run make -f Makefile.ref + +* copy the file src/Linux_All_DBG.OBJ/libjs.so to /usr/lib +and run ldconfig + +Testing +------- + +Please run python jstest.py after installation. This script contains +some tests to corroborate that python bindings installation is OK. + +Known issues +------------ +ctypes fails when loading the library, with old versions of libgcc. We have succeded tests with libgcc version 4.1.2 and higher, and failed with libgcc version libgcc-3.4.6. The exact error message in this case is: + +Traceback (most recent call last): + File "", line 1, in + File "/usr/lib64/python2.5/ctypes/__init__.py", line 340, in __init__ + self._handle = _dlopen(self._name, mode) +OSError: /usr/lib/libjs.so.0: undefined symbol: __clzdi2 diff --git a/scrapy/trunk/scrapy/lib/spidermonkey/__init__.py b/scrapy/trunk/scrapy/lib/spidermonkey/__init__.py new file mode 100644 index 000000000..d29fbe2c5 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/spidermonkey/__init__.py @@ -0,0 +1 @@ +from spidermonkey import * \ No newline at end of file diff --git a/scrapy/trunk/scrapy/lib/spidermonkey/sm_settings.py b/scrapy/trunk/scrapy/lib/spidermonkey/sm_settings.py new file mode 100644 index 000000000..9b37284c3 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/spidermonkey/sm_settings.py @@ -0,0 +1,2 @@ +#defines order in which spidermonkey bin library will be searched +try_libs = ["libmozjs.so.0d", "libjs.so", "libjs.so.0"] \ No newline at end of file diff --git a/scrapy/trunk/scrapy/lib/spidermonkey/spidermonkey.py b/scrapy/trunk/scrapy/lib/spidermonkey/spidermonkey.py new file mode 100644 index 000000000..89e57da84 --- /dev/null +++ b/scrapy/trunk/scrapy/lib/spidermonkey/spidermonkey.py @@ -0,0 +1,2254 @@ +# this is a ctypes-based python wrapper for mozilla spidermonkey + +# ************************************************************************ +# This is an alpha release. It has many known faults, and interfaces will +# change. + +# Note that code listed with JavaScript error messages can be the WRONG +# CODE! Don't take it seriously. +# ************************************************************************ + +# spidermonkey 0.0.2a: Python / JavaScript bridge. +# Copyright (C) 2003 John J. Lee +# Copyright (C) 2006 Leonard Ritter + +# Partly derived from Spidermonkey.xs, Copyright (C) 2001, Claes +# Jacobsson, All Rights Reserved (Perl Artistic License). +# Partly derived from PerlConnect (part of spidermonkey) Copyright (C) +# 1998 Netscape Communications Corporation (GPL). +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +# Security constraints: +# Can't add attributes to Python objects. +# *Can* add JS properties to JS proxies of Python objects, as long as it +# won't hide a Python attribute of the same name. +# Can't assign to Python methods. +# Can't access any Python attribute starting with "_". +# Also, bound Python objects can of course implement their own +# protection, as the Window class does. + +# XXX +# Does JS expect some exceptions on reading a non-existent property? +# Currently, this code has the effect of returning undef in that case, +# which seems odd. Look in O'Reilly book / spidermonkey code. +# Exception propagation. IIRC, I'm waiting for except * to be fixed in +# Pyrex. +# Code listings on JS errors are wrong. +# Review code +# Replace Object with Py_INCREF and just keeping python object in JS +# object (need to think first, though, to check this will work). +# API call return values and bad function arguments! +# Memory management. +# Check strcpy's, malloc's / free's / Destroy*'s. +# Look for and fix any memory leaks. +# To prevent people crashing module, would be nice to make some things +# private. Ask on Pyrex list. +# Make use of JS API security model? +# GC issues (see spidermonkey GC tips page). +# Threading issues (see GC tips again). + +from ctypes import * + +# predefines +size_t = c_ulong +_IO_FILE = c_ulong +__ssize_t = c_int +ssize_t = c_uint + +JSWord = c_long +jsword = JSWord +jsval = jsword + +class JSObject(Structure): + _fields_ = [ + ] +class ObjectJSVALUnion(Union): + _fields_ = [ + ('jsval', jsval), + ('obj', POINTER(JSObject)), + ] + +def OBJECT_TO_JSVAL(obj): + objectJSVALUnion = ObjectJSVALUnion() + objectJSVALUnion.obj = obj + return objectJSVALUnion.jsval + +# defines +EOF = (-1) +FILENAME_MAX = 4096 +FOPEN_MAX = 16 +def JS_BIT(n): + return 1 << n +def JS_BITMASK(n): + return JS_BIT(n) - 1 +def JSVAL_SETTAG(v,t): + if type(v) == c_long: + v = v.value + return v | t +def BOOLEAN_TO_JSVAL(b): + return JSVAL_SETTAG(b << JSVAL_TAGBITS, JSVAL_BOOLEAN) +def JSVAL_INT_POW2(n): + return 1 << n +def INT_TO_JSVAL(i): + return (i << 1) | JSVAL_INT +def JSVAL_IS_PRIMITIVE(v): + return ((not JSVAL_IS_OBJECT(v)) or JSVAL_IS_NULL(v)) +def JSVAL_IS_OBJECT(v): + return (JSVAL_TAG(v) == JSVAL_OBJECT) +def JSVAL_IS_INT(v): + if type(v) == c_long: + v = v.value + return ((v & JSVAL_INT) and (v != JSVAL_VOID)) +def JSVAL_TO_INT(v): + if type(v) == c_long: + v = v.value + return v >> 1 +def JSVAL_IS_DOUBLE(v): + return (JSVAL_TAG(v) == JSVAL_DOUBLE) +def JSVAL_TO_DOUBLE(v): + return cast(JSVAL_TO_GCTHING(v),POINTER(jsdouble)) +def JSVAL_IS_BOOLEAN(v): + return (JSVAL_TAG(v) == JSVAL_BOOLEAN) +def JSVAL_TO_BOOLEAN(v): + if type(v) == c_long: + v = v.value + return v >> JSVAL_TAGBITS +def JSVAL_IS_STRING(v): + return JSVAL_TAG(v) == JSVAL_STRING +def JSVAL_TO_STRING(v): + return cast(JSVAL_TO_GCTHING(v),POINTER(JSString)) +def STRING_TO_JSVAL(str): + return JSVAL_SETTAG(cast(addressof(str),POINTER(jsval)).contents, JSVAL_STRING) +def DOUBLE_TO_JSVAL(dp): + return JSVAL_SETTAG(cast(addressof(dp),POINTER(jsval)).contents, JSVAL_DOUBLE) +def JSVAL_TO_GCTHING(v): + return cast(JSVAL_CLRTAG(v),c_void_p) +def JSVAL_CLRTAG(v): + if type(v) == c_long: + v = v.value + return v & (~JSVAL_TAGMASK) +def JSVAL_TAG(v): + if type(v) == c_long: + v = v.value + return v & JSVAL_TAGMASK +def JSVAL_IS_NULL(v): + if type(v) == c_long: + v = v.value + return v == JSVAL_NULL +def JSVAL_IS_VOID(v): + if type(v) == c_long: + v = v.value + return v == JSVAL_VOID +def JSVAL_TO_OBJECT(v): + return cast(JSVAL_TO_GCTHING(v), POINTER(JSObject)) +JSCLASS_HAS_PRIVATE = (1<<0) +JSCLASS_NEW_ENUMERATE = (1<<1) +JSCLASS_NEW_RESOLVE = (1<<2) +JSCLASS_NEW_RESOLVE_GETS_START = (1<<5) +JSCLASS_NO_OPTIONAL_MEMBERS = 0,0,0,0,0,0,0,0 +JSCLASS_PRIVATE_IS_NSISUPPORTS = (1<<3) +JSCLASS_RESERVED_SLOTS_WIDTH = 8 +JSCLASS_RESERVED_SLOTS_MASK = JS_BITMASK(JSCLASS_RESERVED_SLOTS_WIDTH) +JSCLASS_RESERVED_SLOTS_SHIFT = 8 +JSCLASS_SHARE_ALL_PROPERTIES = (1<<4) +JSFUN_BOUND_METHOD = 0x40 +JSFUN_FLAGS_MASK = 0xf8 +JSFUN_HEAVYWEIGHT = 0x80 +JSFUN_LAMBDA = 0x08 +JSOPTION_COMPILE_N_GO = JS_BIT(4) +JSOPTION_PRIVATE_IS_NSISUPPORTS = JS_BIT(3) +JSOPTION_STRICT = JS_BIT(0) +JSOPTION_VAROBJFIX = JS_BIT(2) +JSOPTION_WERROR = JS_BIT(1) +JSPROP_ENUMERATE = 0x01 +JSPROP_EXPORTED = 0x08 +JSPROP_GETTER = 0x10 +JSPROP_INDEX = 0x80 +JSPROP_PERMANENT = 0x04 +JSPROP_READONLY = 0x02 +JSPROP_SETTER = 0x20 +JSPROP_SHARED = 0x40 +JSREG_FOLD = 0x01 +JSREG_GLOB = 0x02 +JSREG_MULTILINE = 0x04 +JSREPORT_ERROR = 0x0 +JSREPORT_EXCEPTION = 0x2 +JSREPORT_STRICT = 0x4 +JSREPORT_WARNING = 0x1 +JSRESOLVE_ASSIGNING = 0x02 +JSRESOLVE_QUALIFIED = 0x01 +JSVAL_TAGBITS = 3 +JSVAL_ALIGN = JS_BIT(JSVAL_TAGBITS) +JSVAL_BOOLEAN = 0x6 +JSVAL_DOUBLE = 0x2 +JS_FALSE = 0 +JSVAL_FALSE = BOOLEAN_TO_JSVAL(JS_FALSE) +JSVAL_INT = 0x1 +JSVAL_INT_BITS = 31 +JSVAL_INT_MAX = (JSVAL_INT_POW2(30) - 1) +JSVAL_INT_MIN = (1 - JSVAL_INT_POW2(30)) +JSVAL_NULL = 0 +JSVAL_OBJECT = 0x0 +JSVAL_ONE = INT_TO_JSVAL(1) +JSVAL_STRING = 0x4 +JSVAL_TAGMASK = JS_BITMASK(JSVAL_TAGBITS) +JS_TRUE = 1 +JSVAL_TRUE = BOOLEAN_TO_JSVAL(JS_TRUE) +JSVAL_VOID = INT_TO_JSVAL(0 - JSVAL_INT_POW2(30)) +JSVAL_ZERO = INT_TO_JSVAL(0) +JS_DONT_PRETTY_PRINT = (0x8000) +JS_MAP_GCROOT_NEXT = 0 +JS_MAP_GCROOT_REMOVE = 2 +JS_MAP_GCROOT_STOP = 1 +SEEK_CUR = 1 +SEEK_END = 2 +SEEK_SET = 0 +TMP_MAX = 238328 + +# using file "/usr/include/smjs/jslong.h" +# using file "/usr/include/smjs/jscompat.h" +# using file "/usr/include/smjs/jspubtd.h" +# using file "/usr/include/smjs/jsapi.h" +JSUintn = c_uint + +uintN = JSUintn + +class JSErrorFormatString(Structure): + _fields_ = [ + ('format', c_char_p), + ('argCount', uintN), + ] + +JSErrorFormatString = JSErrorFormatString + +class JSContext(Structure): + _fields_ = [ + ] + +JSContext = JSContext + +JSIntn = c_int + +JSBool = JSIntn + +JSInt32 = c_int + +int32 = JSInt32 + +jsrefcount = int32 + +class JSPrincipals(Structure): + _fields_ = [ + ('codebase', c_char_p), + ('getPrincipalArray', CFUNCTYPE(c_void_p,POINTER(JSContext),POINTER('JSPrincipals'))), + ('globalPrivilegesEnabled', CFUNCTYPE(JSBool,POINTER(JSContext),POINTER('JSPrincipals'))), + ('refcount', jsrefcount), + ('destroy', CFUNCTYPE(None,POINTER(JSContext),POINTER('JSPrincipals'))), + ] + +JSPrincipals = JSPrincipals + +jsid = jsword + +class JSProperty(Structure): + _fields_ = [ + ('id', jsid), + ] + +JSProperty = JSProperty + +class JSObjectMap(Structure): + _fields_ = [ + ] + +JSObjectMap = JSObjectMap + +class JSXDRState(Structure): + _fields_ = [ + ] + +JSXDRState = JSXDRState + +class JSObjectOps(Structure): + _fields_ = [] + +JSUint32 = c_uint + +uint32 = JSUint32 + +JSObject = JSObject + +JSPropertyOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)) + +JSEnumerateOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject)) + +JSResolveOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long) + +# enumeration JSType +JSTYPE_VOID = 0 +JSTYPE_OBJECT = 1 +JSTYPE_FUNCTION = 2 +JSTYPE_STRING = 3 +JSTYPE_NUMBER = 4 +JSTYPE_BOOLEAN = 5 +JSTYPE_LIMIT = 6 + +JSConvertOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_int,POINTER(jsval)) + +JSFinalizeOp = CFUNCTYPE(None,POINTER(JSContext),POINTER(JSObject)) + +class JSClass(Structure): + _fields_ = [] + +JSGetObjectOps = CFUNCTYPE(POINTER(JSObjectOps),POINTER(JSContext),POINTER(JSClass)) + +# enumeration JSAccessMode +JSACC_PROTO = 0 +JSACC_PARENT = 1 +JSACC_IMPORT = 2 +JSACC_WATCH = 3 +JSACC_READ = 4 +JSACC_WRITE = 8 +JSACC_LIMIT = 9 + +JSCheckAccessOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,c_int,POINTER(jsval)) + +JSNative = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_uint,POINTER(jsval),POINTER(jsval)) + +JSXDRObjectOp = CFUNCTYPE(JSBool,POINTER(JSXDRState),POINTER(POINTER(JSObject))) + +JSHasInstanceOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(JSBool)) + +JSMarkOp = CFUNCTYPE(uint32,POINTER(JSContext),POINTER(JSObject),c_void_p) + +class JSClass(Structure): + _fields_ = [ + ('name', c_char_p), + ('flags', uint32), + ('addProperty', JSPropertyOp), + ('delProperty', JSPropertyOp), + ('getProperty', JSPropertyOp), + ('setProperty', JSPropertyOp), + ('enumerate', JSEnumerateOp), + ('resolve', JSResolveOp), + ('convert', JSConvertOp), + ('finalize', JSFinalizeOp), + ('getObjectOps', JSGetObjectOps), + ('checkAccess', JSCheckAccessOp), + ('call', JSNative), + ('construct', JSNative), + ('xdrObject', JSXDRObjectOp), + ('hasInstance', JSHasInstanceOp), + ('mark', JSMarkOp), + ('spare', jsword), + ] + +JSClass = JSClass + +JSNewObjectMapOp = CFUNCTYPE(POINTER(JSObjectMap),POINTER(JSContext),c_int,POINTER(JSObjectOps),POINTER(JSClass),POINTER(JSObject)) + +JSObjectMapOp = CFUNCTYPE(None,POINTER(JSContext),POINTER(JSObjectMap)) + +JSLookupPropOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(POINTER(JSObject)),POINTER(POINTER(JSProperty))) + +JSDefinePropOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,c_long,CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),c_uint,POINTER(POINTER(JSProperty))) + +JSPropertyIdOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)) + +JSAttributesOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(JSProperty),POINTER(uintN)) + +# enumeration JSIterateOp +JSENUMERATE_INIT = 0 +JSENUMERATE_NEXT = 1 +JSENUMERATE_DESTROY = 2 + +JSNewEnumerateOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_int,POINTER(jsval),POINTER(jsid)) + +JSCheckAccessIdOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,c_int,POINTER(jsval),POINTER(uintN)) + +JSObjectOp = CFUNCTYPE(POINTER(JSObject),POINTER(JSContext),POINTER(JSObject)) + +JSPropertyRefOp = CFUNCTYPE(None,POINTER(JSContext),POINTER(JSObject),POINTER(JSProperty)) + +JSSetObjectSlotOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_uint,POINTER(JSObject)) + +JSGetRequiredSlotOp = CFUNCTYPE(jsval,POINTER(JSContext),POINTER(JSObject),c_uint) + +JSSetRequiredSlotOp = CFUNCTYPE(None,POINTER(JSContext),POINTER(JSObject),c_uint,c_long) + +class JSObjectOps(Structure): + _fields_ = [ + ('newObjectMap', JSNewObjectMapOp), + ('destroyObjectMap', JSObjectMapOp), + ('lookupProperty', JSLookupPropOp), + ('defineProperty', JSDefinePropOp), + ('getProperty', JSPropertyIdOp), + ('setProperty', JSPropertyIdOp), + ('getAttributes', JSAttributesOp), + ('setAttributes', JSAttributesOp), + ('deleteProperty', JSPropertyIdOp), + ('defaultValue', JSConvertOp), + ('enumerate', JSNewEnumerateOp), + ('checkAccess', JSCheckAccessIdOp), + ('thisObject', JSObjectOp), + ('dropProperty', JSPropertyRefOp), + ('call', JSNative), + ('construct', JSNative), + ('xdrObject', JSXDRObjectOp), + ('hasInstance', JSHasInstanceOp), + ('setProto', JSSetObjectSlotOp), + ('setParent', JSSetObjectSlotOp), + ('mark', JSMarkOp), + ('clear', JSFinalizeOp), + ('getRequiredSlot', JSGetRequiredSlotOp), + ('setRequiredSlot', JSSetRequiredSlotOp), + ] + +JSObjectOps = JSObjectOps + +JSUint16 = c_ushort + +uint16 = JSUint16 + +jschar = uint16 + +class JSErrorReport(Structure): + _fields_ = [ + ('filename', c_char_p), + ('lineno', uintN), + ('linebuf', c_char_p), + ('tokenptr', c_char_p), + ('uclinebuf', POINTER(jschar)), + ('uctokenptr', POINTER(jschar)), + ('flags', uintN), + ('errorNumber', uintN), + ('ucmessage', POINTER(jschar)), + ('messageArgs', POINTER(POINTER(jschar))), + ] + +class JSString(Structure): + _fields_ = [ + ] + +JSString = JSString + +JSLocaleToUpperCase = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSString),POINTER(jsval)) + +JSLocaleToLowerCase = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSString),POINTER(jsval)) + +JSLocaleCompare = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSString),POINTER(JSString),POINTER(jsval)) + +class JSLocaleCallbacks(Structure): + _fields_ = [ + ('localeToUpperCase', JSLocaleToUpperCase), + ('localeToLowerCase', JSLocaleToLowerCase), + ('localeCompare', JSLocaleCompare), + ] + +JSUint8 = c_ubyte + +uint8 = JSUint8 + +class JSFunctionSpec(Structure): + _fields_ = [ + ('name', c_char_p), + ('call', JSNative), + ('nargs', uint8), + ('flags', uint8), + ('extra', uint16), + ] + +JSInt8 = c_char + +int8 = JSInt8 + +class JSPropertySpec(Structure): + _fields_ = [ + ('name', c_char_p), + ('tinyid', int8), + ('flags', uint8), + ('getter', JSPropertyOp), + ('setter', JSPropertyOp), + ] + +JSFloat64 = c_double + +float64 = JSFloat64 + +jsdouble = float64 + +class JSConstDoubleSpec(Structure): + _fields_ = [ + ('dval', jsdouble), + ('name', c_char_p), + ('flags', uint8), + ('spare', uint8*3), + ] + +jsint = int32 + +class JSIdArray(Structure): + _fields_ = [ + ('length', jsint), + ('vector', jsid*1), + ] + +intN = JSIntn + +JSErrorReport = JSErrorReport + +class JSScript(Structure): + _fields_ = [ + ] + +JSScript = JSScript + +# enumeration JSGCStatus +JSGC_BEGIN = 0 +JSGC_END = 1 +JSGC_MARK_END = 2 +JSGC_FINALIZE_END = 3 + +JSConstDoubleSpec = JSConstDoubleSpec + +JSPropertySpec = JSPropertySpec + +jsuint = uint32 + +JSIdArray = JSIdArray + +class JSRuntime(Structure): + _fields_ = [ + ] + +JSRuntime = JSRuntime + +JSFunctionSpec = JSFunctionSpec + +class JSFunction(Structure): + _fields_ = [ + ] + +JSFunction = JSFunction + +JSLocaleCallbacks = JSLocaleCallbacks + +class JSExceptionState(Structure): + _fields_ = [ + ] + +JSExceptionState = JSExceptionState + +JSInt64 = c_longlong + +from sm_settings import try_libs +for lib in try_libs: + try: + libsmjs = CDLL(lib) + break + except OSError: + if lib == try_libs[-1]: + raise ImportError("The spidermonkey C library is not installed") + +JSLL_MaxInt = libsmjs.JSLL_MaxInt +JSLL_MaxInt.restype = JSInt64 +JSLL_MaxInt.argtypes = [] + +JSLL_MinInt = libsmjs.JSLL_MinInt +JSLL_MinInt.restype = JSInt64 +JSLL_MinInt.argtypes = [] + +JSLL_Zero = libsmjs.JSLL_Zero +JSLL_Zero.restype = JSInt64 +JSLL_Zero.argtypes = [] + +JSUword = c_ulong + +jsuword = JSUword + +float32 = c_float + +# enumeration Js-1.7.0.tar.gzSVersion +JSVERSION_1_0 = 100 +JSVERSION_1_1 = 110 +JSVERSION_1_2 = 120 +JSVERSION_1_3 = 130 +JSVERSION_1_4 = 140 +JSVERSION_ECMA_3 = 148 +JSVERSION_1_5 = 150 +JSVERSION_DEFAULT = 0 +JSVERSION_UNKNOWN = -1 + +JSVersion = c_int + +JSType = c_int + +JSAccessMode = c_int + +JSIterateOp = c_int + +JSTaskState = JSRuntime + +JSNewResolveOp = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,c_uint,POINTER(POINTER(JSObject))) + +JSStringFinalizeOp = CFUNCTYPE(None,POINTER(JSContext),POINTER(JSString)) + +JSGCStatus = c_int + +JSGCCallback = CFUNCTYPE(JSBool,POINTER(JSContext),c_int) + +JSBranchCallback = CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSScript)) + +JSErrorReporter = CFUNCTYPE(None,POINTER(JSContext),c_char_p,POINTER(JSErrorReport)) + +JSErrorCallback = CFUNCTYPE(POINTER(JSErrorFormatString),c_void_p,c_char_p,c_uint) + +JSPrincipalsTranscoder = CFUNCTYPE(JSBool,POINTER(JSXDRState),POINTER(POINTER(JSPrincipals))) + +JSObjectPrincipalsFinder = CFUNCTYPE(POINTER(JSPrincipals),POINTER(JSContext),POINTER(JSObject)) + +int64 = JSInt64 + +JS_Now = libsmjs.JS_Now +JS_Now.restype = int64 +JS_Now.argtypes = [] + +JS_GetNaNValue = libsmjs.JS_GetNaNValue +JS_GetNaNValue.restype = jsval +JS_GetNaNValue.argtypes = [POINTER(JSContext)] + +JS_GetNegativeInfinityValue = libsmjs.JS_GetNegativeInfinityValue +JS_GetNegativeInfinityValue.restype = jsval +JS_GetNegativeInfinityValue.argtypes = [POINTER(JSContext)] + +JS_GetPositiveInfinityValue = libsmjs.JS_GetPositiveInfinityValue +JS_GetPositiveInfinityValue.restype = jsval +JS_GetPositiveInfinityValue.argtypes = [POINTER(JSContext)] + +JS_GetEmptyStringValue = libsmjs.JS_GetEmptyStringValue +JS_GetEmptyStringValue.restype = jsval +JS_GetEmptyStringValue.argtypes = [POINTER(JSContext)] + +JS_ConvertArguments = libsmjs.JS_ConvertArguments +JS_ConvertArguments.restype = JSBool +JS_ConvertArguments.argtypes = [POINTER(JSContext),c_uint,POINTER(jsval),c_char_p] + +JS_PushArguments = libsmjs.JS_PushArguments +JS_PushArguments.restype = POINTER(jsval) +JS_PushArguments.argtypes = [POINTER(JSContext),POINTER(c_void_p),c_char_p] + +JS_PopArguments = libsmjs.JS_PopArguments +JS_PopArguments.restype = None +JS_PopArguments.argtypes = [POINTER(JSContext),c_void_p] + +JS_ConvertValue = libsmjs.JS_ConvertValue +JS_ConvertValue.restype = JSBool +JS_ConvertValue.argtypes = [POINTER(JSContext),c_long,c_int,POINTER(jsval)] + +JS_ValueToObject = libsmjs.JS_ValueToObject +JS_ValueToObject.restype = JSBool +JS_ValueToObject.argtypes = [POINTER(JSContext),c_long,POINTER(POINTER(JSObject))] + +JS_ValueToFunction = libsmjs.JS_ValueToFunction +JS_ValueToFunction.restype = POINTER(JSFunction) +JS_ValueToFunction.argtypes = [POINTER(JSContext),c_long] + +JS_ValueToConstructor = libsmjs.JS_ValueToConstructor +JS_ValueToConstructor.restype = POINTER(JSFunction) +JS_ValueToConstructor.argtypes = [POINTER(JSContext),c_long] + +JS_ValueToString = libsmjs.JS_ValueToString +JS_ValueToString.restype = POINTER(JSString) +JS_ValueToString.argtypes = [POINTER(JSContext),c_long] + +JS_ValueToNumber = libsmjs.JS_ValueToNumber +JS_ValueToNumber.restype = JSBool +JS_ValueToNumber.argtypes = [POINTER(JSContext),c_long,POINTER(jsdouble)] + +JS_ValueToECMAInt32 = libsmjs.JS_ValueToECMAInt32 +JS_ValueToECMAInt32.restype = JSBool +JS_ValueToECMAInt32.argtypes = [POINTER(JSContext),c_long,POINTER(int32)] + +JS_ValueToECMAUint32 = libsmjs.JS_ValueToECMAUint32 +JS_ValueToECMAUint32.restype = JSBool +JS_ValueToECMAUint32.argtypes = [POINTER(JSContext),c_long,POINTER(uint32)] + +JS_ValueToInt32 = libsmjs.JS_ValueToInt32 +JS_ValueToInt32.restype = JSBool +JS_ValueToInt32.argtypes = [POINTER(JSContext),c_long,POINTER(int32)] + +JS_ValueToUint16 = libsmjs.JS_ValueToUint16 +JS_ValueToUint16.restype = JSBool +JS_ValueToUint16.argtypes = [POINTER(JSContext),c_long,POINTER(uint16)] + +JS_ValueToBoolean = libsmjs.JS_ValueToBoolean +JS_ValueToBoolean.restype = JSBool +JS_ValueToBoolean.argtypes = [POINTER(JSContext),c_long,POINTER(JSBool)] + +JS_TypeOfValue = libsmjs.JS_TypeOfValue +JS_TypeOfValue.restype = JSType +JS_TypeOfValue.argtypes = [POINTER(JSContext),c_long] + +JS_GetTypeName = libsmjs.JS_GetTypeName +JS_GetTypeName.restype = c_char_p +JS_GetTypeName.argtypes = [POINTER(JSContext),c_int] + +JS_Init = libsmjs.JS_Init +JS_Init.restype = POINTER(JSRuntime) +JS_Init.argtypes = [c_uint] + +JS_Finish = libsmjs.JS_Finish +JS_Finish.restype = None +JS_Finish.argtypes = [POINTER(JSRuntime)] + +JS_ShutDown = libsmjs.JS_ShutDown +JS_ShutDown.restype = None +JS_ShutDown.argtypes = [] + +JS_GetRuntimePrivate = libsmjs.JS_GetRuntimePrivate +JS_GetRuntimePrivate.restype = c_void_p +JS_GetRuntimePrivate.argtypes = [POINTER(JSRuntime)] + +JS_SetRuntimePrivate = libsmjs.JS_SetRuntimePrivate +JS_SetRuntimePrivate.restype = None +JS_SetRuntimePrivate.argtypes = [POINTER(JSRuntime),c_void_p] + +JS_Lock = libsmjs.JS_Lock +JS_Lock.restype = None +JS_Lock.argtypes = [POINTER(JSRuntime)] + +JS_Unlock = libsmjs.JS_Unlock +JS_Unlock.restype = None +JS_Unlock.argtypes = [POINTER(JSRuntime)] + +JS_NewContext = libsmjs.JS_NewContext +JS_NewContext.restype = POINTER(JSContext) +JS_NewContext.argtypes = [POINTER(JSRuntime),c_uint] + +JS_DestroyContext = libsmjs.JS_DestroyContext +JS_DestroyContext.restype = None +JS_DestroyContext.argtypes = [POINTER(JSContext)] + +JS_DestroyContextNoGC = libsmjs.JS_DestroyContextNoGC +JS_DestroyContextNoGC.restype = None +JS_DestroyContextNoGC.argtypes = [POINTER(JSContext)] + +JS_DestroyContextMaybeGC = libsmjs.JS_DestroyContextMaybeGC +JS_DestroyContextMaybeGC.restype = None +JS_DestroyContextMaybeGC.argtypes = [POINTER(JSContext)] + +JS_GetContextPrivate = libsmjs.JS_GetContextPrivate +JS_GetContextPrivate.restype = c_void_p +JS_GetContextPrivate.argtypes = [POINTER(JSContext)] + +JS_SetContextPrivate = libsmjs.JS_SetContextPrivate +JS_SetContextPrivate.restype = None +JS_SetContextPrivate.argtypes = [POINTER(JSContext),c_void_p] + +JS_GetRuntime = libsmjs.JS_GetRuntime +JS_GetRuntime.restype = POINTER(JSRuntime) +JS_GetRuntime.argtypes = [POINTER(JSContext)] + +JS_ContextIterator = libsmjs.JS_ContextIterator +JS_ContextIterator.restype = POINTER(JSContext) +JS_ContextIterator.argtypes = [POINTER(JSRuntime),POINTER(POINTER(JSContext))] + +JS_GetVersion = libsmjs.JS_GetVersion +JS_GetVersion.restype = JSVersion +JS_GetVersion.argtypes = [POINTER(JSContext)] + +JS_SetVersion = libsmjs.JS_SetVersion +JS_SetVersion.restype = JSVersion +JS_SetVersion.argtypes = [POINTER(JSContext),c_int] + +JS_VersionToString = libsmjs.JS_VersionToString +JS_VersionToString.restype = c_char_p +JS_VersionToString.argtypes = [c_int] + +JS_StringToVersion = libsmjs.JS_StringToVersion +JS_StringToVersion.restype = JSVersion +JS_StringToVersion.argtypes = [c_char_p] + +JS_GetOptions = libsmjs.JS_GetOptions +JS_GetOptions.restype = uint32 +JS_GetOptions.argtypes = [POINTER(JSContext)] + +JS_SetOptions = libsmjs.JS_SetOptions +JS_SetOptions.restype = uint32 +JS_SetOptions.argtypes = [POINTER(JSContext),c_uint] + +JS_ToggleOptions = libsmjs.JS_ToggleOptions +JS_ToggleOptions.restype = uint32 +JS_ToggleOptions.argtypes = [POINTER(JSContext),c_uint] + +JS_GetImplementationVersion = libsmjs.JS_GetImplementationVersion +JS_GetImplementationVersion.restype = c_char_p +JS_GetImplementationVersion.argtypes = [] + +JS_GetGlobalObject = libsmjs.JS_GetGlobalObject +JS_GetGlobalObject.restype = POINTER(JSObject) +JS_GetGlobalObject.argtypes = [POINTER(JSContext)] + +JS_SetGlobalObject = libsmjs.JS_SetGlobalObject +JS_SetGlobalObject.restype = None +JS_SetGlobalObject.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_InitStandardClasses = libsmjs.JS_InitStandardClasses +JS_InitStandardClasses.restype = JSBool +JS_InitStandardClasses.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_ResolveStandardClass = libsmjs.JS_ResolveStandardClass +JS_ResolveStandardClass.restype = JSBool +JS_ResolveStandardClass.argtypes = [POINTER(JSContext),POINTER(JSObject),c_long,POINTER(JSBool)] + +JS_EnumerateStandardClasses = libsmjs.JS_EnumerateStandardClasses +JS_EnumerateStandardClasses.restype = JSBool +JS_EnumerateStandardClasses.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_GetScopeChain = libsmjs.JS_GetScopeChain +JS_GetScopeChain.restype = POINTER(JSObject) +JS_GetScopeChain.argtypes = [POINTER(JSContext)] + +JS_malloc = libsmjs.JS_malloc +JS_malloc.restype = c_void_p +JS_malloc.argtypes = [POINTER(JSContext),c_uint] + +JS_realloc = libsmjs.JS_realloc +JS_realloc.restype = c_void_p +JS_realloc.argtypes = [POINTER(JSContext),c_void_p,c_uint] + +JS_free = libsmjs.JS_free +JS_free.restype = None +JS_free.argtypes = [POINTER(JSContext),c_void_p] + +JS_strdup = libsmjs.JS_strdup +JS_strdup.restype = c_char_p +JS_strdup.argtypes = [POINTER(JSContext),c_char_p] + +JS_NewDouble = libsmjs.JS_NewDouble +JS_NewDouble.restype = POINTER(jsdouble) +JS_NewDouble.argtypes = [POINTER(JSContext),c_double] + +JS_NewDoubleValue = libsmjs.JS_NewDoubleValue +JS_NewDoubleValue.restype = JSBool +JS_NewDoubleValue.argtypes = [POINTER(JSContext),c_double,POINTER(jsval)] + +JS_NewNumberValue = libsmjs.JS_NewNumberValue +JS_NewNumberValue.restype = JSBool +JS_NewNumberValue.argtypes = [POINTER(JSContext),c_double,POINTER(jsval)] + +JS_AddRoot = libsmjs.JS_AddRoot +JS_AddRoot.restype = JSBool +JS_AddRoot.argtypes = [POINTER(JSContext),c_void_p] + +JS_AddNamedRoot = libsmjs.JS_AddNamedRoot +JS_AddNamedRoot.restype = JSBool +JS_AddNamedRoot.argtypes = [POINTER(JSContext),c_void_p,c_char_p] + +JS_AddNamedRootRT = libsmjs.JS_AddNamedRootRT +JS_AddNamedRootRT.restype = JSBool +JS_AddNamedRootRT.argtypes = [POINTER(JSRuntime),c_void_p,c_char_p] + +JS_RemoveRoot = libsmjs.JS_RemoveRoot +JS_RemoveRoot.restype = JSBool +JS_RemoveRoot.argtypes = [POINTER(JSContext),c_void_p] + +JS_RemoveRootRT = libsmjs.JS_RemoveRootRT +JS_RemoveRootRT.restype = JSBool +JS_RemoveRootRT.argtypes = [POINTER(JSRuntime),c_void_p] + +JS_ClearNewbornRoots = libsmjs.JS_ClearNewbornRoots +JS_ClearNewbornRoots.restype = None +JS_ClearNewbornRoots.argtypes = [POINTER(JSContext)] + +JSGCRootMapFun = CFUNCTYPE(intN,c_void_p,c_char_p,c_void_p) + +JS_MapGCRoots = libsmjs.JS_MapGCRoots +JS_MapGCRoots.restype = uint32 +JS_MapGCRoots.argtypes = [POINTER(JSRuntime),CFUNCTYPE(intN,c_void_p,c_char_p,c_void_p),c_void_p] + +JS_LockGCThing = libsmjs.JS_LockGCThing +JS_LockGCThing.restype = JSBool +JS_LockGCThing.argtypes = [POINTER(JSContext),c_void_p] + +JS_LockGCThingRT = libsmjs.JS_LockGCThingRT +JS_LockGCThingRT.restype = JSBool +JS_LockGCThingRT.argtypes = [POINTER(JSRuntime),c_void_p] + +JS_UnlockGCThing = libsmjs.JS_UnlockGCThing +JS_UnlockGCThing.restype = JSBool +JS_UnlockGCThing.argtypes = [POINTER(JSContext),c_void_p] + +JS_UnlockGCThingRT = libsmjs.JS_UnlockGCThingRT +JS_UnlockGCThingRT.restype = JSBool +JS_UnlockGCThingRT.argtypes = [POINTER(JSRuntime),c_void_p] + +JS_MarkGCThing = libsmjs.JS_MarkGCThing +JS_MarkGCThing.restype = None +JS_MarkGCThing.argtypes = [POINTER(JSContext),c_void_p,c_char_p,c_void_p] + +JS_GC = libsmjs.JS_GC +JS_GC.restype = None +JS_GC.argtypes = [POINTER(JSContext)] + +JS_MaybeGC = libsmjs.JS_MaybeGC +JS_MaybeGC.restype = None +JS_MaybeGC.argtypes = [POINTER(JSContext)] + +JS_SetGCCallback = libsmjs.JS_SetGCCallback +JS_SetGCCallback.restype = JSGCCallback +JS_SetGCCallback.argtypes = [POINTER(JSContext),CFUNCTYPE(JSBool,POINTER(JSContext),c_int)] + +JS_SetGCCallbackRT = libsmjs.JS_SetGCCallbackRT +JS_SetGCCallbackRT.restype = JSGCCallback +JS_SetGCCallbackRT.argtypes = [POINTER(JSRuntime),CFUNCTYPE(JSBool,POINTER(JSContext),c_int)] + +JS_IsAboutToBeFinalized = libsmjs.JS_IsAboutToBeFinalized +JS_IsAboutToBeFinalized.restype = JSBool +JS_IsAboutToBeFinalized.argtypes = [POINTER(JSContext),c_void_p] + +JS_AddExternalStringFinalizer = libsmjs.JS_AddExternalStringFinalizer +JS_AddExternalStringFinalizer.restype = intN +JS_AddExternalStringFinalizer.argtypes = [CFUNCTYPE(None,POINTER(JSContext),POINTER(JSString))] + +JS_RemoveExternalStringFinalizer = libsmjs.JS_RemoveExternalStringFinalizer +JS_RemoveExternalStringFinalizer.restype = intN +JS_RemoveExternalStringFinalizer.argtypes = [CFUNCTYPE(None,POINTER(JSContext),POINTER(JSString))] + +JS_NewExternalString = libsmjs.JS_NewExternalString +JS_NewExternalString.restype = POINTER(JSString) +JS_NewExternalString.argtypes = [POINTER(JSContext),POINTER(jschar),c_uint,c_int] + +JS_GetExternalStringGCType = libsmjs.JS_GetExternalStringGCType +JS_GetExternalStringGCType.restype = intN +JS_GetExternalStringGCType.argtypes = [POINTER(JSRuntime),POINTER(JSString)] + +JS_SetThreadStackLimit = libsmjs.JS_SetThreadStackLimit +JS_SetThreadStackLimit.restype = None +JS_SetThreadStackLimit.argtypes = [POINTER(JSContext),c_ulong] + +JS_DestroyIdArray = libsmjs.JS_DestroyIdArray +JS_DestroyIdArray.restype = None +JS_DestroyIdArray.argtypes = [POINTER(JSContext),POINTER(JSIdArray)] + +JS_ValueToId = libsmjs.JS_ValueToId +JS_ValueToId.restype = JSBool +JS_ValueToId.argtypes = [POINTER(JSContext),c_long,POINTER(jsid)] + +JS_IdToValue = libsmjs.JS_IdToValue +JS_IdToValue.restype = JSBool +JS_IdToValue.argtypes = [POINTER(JSContext),c_long,POINTER(jsval)] + +JS_PropertyStub = JSPropertyOp(libsmjs.JS_PropertyStub) +#JS_PropertyStub.restype = JSBool +#JS_PropertyStub.argtypes = [POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)] + +JS_EnumerateStub = JSEnumerateOp(libsmjs.JS_EnumerateStub) +#JS_EnumerateStub.restype = JSBool +#JS_EnumerateStub.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_ResolveStub = JSResolveOp(libsmjs.JS_ResolveStub) +#JS_ResolveStub.restype = JSBool +#JS_ResolveStub.argtypes = [POINTER(JSContext),POINTER(JSObject),c_long] + +JS_ConvertStub = JSConvertOp(libsmjs.JS_ConvertStub) +#JS_ConvertStub.restype = JSBool +#JS_ConvertStub.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int,POINTER(jsval)] + +JS_FinalizeStub = JSFinalizeOp(libsmjs.JS_FinalizeStub) +#JS_FinalizeStub.restype = None +#JS_FinalizeStub.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_InitClass = libsmjs.JS_InitClass +JS_InitClass.restype = POINTER(JSObject) +JS_InitClass.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSObject),POINTER(JSClass),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_uint,POINTER(jsval),POINTER(jsval)),c_uint,POINTER(JSPropertySpec),POINTER(JSFunctionSpec),POINTER(JSPropertySpec),POINTER(JSFunctionSpec)] + +JS_GetClass = libsmjs.JS_GetClass +JS_GetClass.restype = POINTER(JSClass) +JS_GetClass.argtypes = [POINTER(JSContext),POINTER(JSObject)] +#JS_GetClass.argtypes = [POINTER(JSObject)] + +JS_InstanceOf = libsmjs.JS_InstanceOf +JS_InstanceOf.restype = JSBool +JS_InstanceOf.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSClass),POINTER(jsval)] + +JS_GetPrivate = libsmjs.JS_GetPrivate +JS_GetPrivate.restype = c_void_p +JS_GetPrivate.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_SetPrivate = libsmjs.JS_SetPrivate +JS_SetPrivate.restype = JSBool +JS_SetPrivate.argtypes = [POINTER(JSContext),POINTER(JSObject),c_void_p] + +JS_GetInstancePrivate = libsmjs.JS_GetInstancePrivate +JS_GetInstancePrivate.restype = c_void_p +JS_GetInstancePrivate.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSClass),POINTER(jsval)] + +JS_GetPrototype = libsmjs.JS_GetPrototype +JS_GetPrototype.restype = POINTER(JSObject) +JS_GetPrototype.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_SetPrototype = libsmjs.JS_SetPrototype +JS_SetPrototype.restype = JSBool +JS_SetPrototype.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSObject)] + +JS_GetParent = libsmjs.JS_GetParent +JS_GetParent.restype = POINTER(JSObject) +JS_GetParent.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_SetParent = libsmjs.JS_SetParent +JS_SetParent.restype = JSBool +JS_SetParent.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSObject)] + +JS_GetConstructor = libsmjs.JS_GetConstructor +JS_GetConstructor.restype = POINTER(JSObject) +JS_GetConstructor.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_GetObjectId = libsmjs.JS_GetObjectId +JS_GetObjectId.restype = JSBool +JS_GetObjectId.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jsid)] + +JS_NewObject = libsmjs.JS_NewObject +JS_NewObject.restype = POINTER(JSObject) +JS_NewObject.argtypes = [POINTER(JSContext),POINTER(JSClass),POINTER(JSObject),POINTER(JSObject)] + +JS_SealObject = libsmjs.JS_SealObject +JS_SealObject.restype = JSBool +JS_SealObject.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int] + +JS_ConstructObject = libsmjs.JS_ConstructObject +JS_ConstructObject.restype = POINTER(JSObject) +JS_ConstructObject.argtypes = [POINTER(JSContext),POINTER(JSClass),POINTER(JSObject),POINTER(JSObject)] + +JS_ConstructObjectWithArguments = libsmjs.JS_ConstructObjectWithArguments +JS_ConstructObjectWithArguments.restype = POINTER(JSObject) +JS_ConstructObjectWithArguments.argtypes = [POINTER(JSContext),POINTER(JSClass),POINTER(JSObject),POINTER(JSObject),c_uint,POINTER(jsval)] + +JS_DefineObject = libsmjs.JS_DefineObject +JS_DefineObject.restype = POINTER(JSObject) +JS_DefineObject.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(JSClass),POINTER(JSObject),c_uint] + +JS_DefineConstDoubles = libsmjs.JS_DefineConstDoubles +JS_DefineConstDoubles.restype = JSBool +JS_DefineConstDoubles.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSConstDoubleSpec)] + +JS_DefineProperties = libsmjs.JS_DefineProperties +JS_DefineProperties.restype = JSBool +JS_DefineProperties.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSPropertySpec)] + +JS_DefineProperty = libsmjs.JS_DefineProperty +JS_DefineProperty.restype = JSBool +#JS_DefineProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_long,CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),c_uint] +JS_DefineProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_long,c_void_p,c_void_p,c_uint] + +JS_GetPropertyAttributes = libsmjs.JS_GetPropertyAttributes +JS_GetPropertyAttributes.restype = JSBool +JS_GetPropertyAttributes.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(uintN),POINTER(JSBool)] + +JS_SetPropertyAttributes = libsmjs.JS_SetPropertyAttributes +JS_SetPropertyAttributes.restype = JSBool +JS_SetPropertyAttributes.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_uint,POINTER(JSBool)] + +JS_DefinePropertyWithTinyId = libsmjs.JS_DefinePropertyWithTinyId +JS_DefinePropertyWithTinyId.restype = JSBool +JS_DefinePropertyWithTinyId.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_char,c_long,CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),c_uint] + +JS_AliasProperty = libsmjs.JS_AliasProperty +JS_AliasProperty.restype = JSBool +JS_AliasProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_char_p] + +JS_LookupProperty = libsmjs.JS_LookupProperty +JS_LookupProperty.restype = JSBool +JS_LookupProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(jsval)] + +JS_GetProperty = libsmjs.JS_GetProperty +JS_GetProperty.restype = JSBool +JS_GetProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(jsval)] + +JS_SetProperty = libsmjs.JS_SetProperty +JS_SetProperty.restype = JSBool +JS_SetProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(jsval)] + +JS_DeleteProperty = libsmjs.JS_DeleteProperty +JS_DeleteProperty.restype = JSBool +JS_DeleteProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p] + +JS_DeleteProperty2 = libsmjs.JS_DeleteProperty2 +JS_DeleteProperty2.restype = JSBool +JS_DeleteProperty2.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(jsval)] + +JS_DefineUCProperty = libsmjs.JS_DefineUCProperty +JS_DefineUCProperty.restype = JSBool +JS_DefineUCProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,c_long,CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),c_uint] + +JS_GetUCPropertyAttributes = libsmjs.JS_GetUCPropertyAttributes +JS_GetUCPropertyAttributes.restype = JSBool +JS_GetUCPropertyAttributes.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,POINTER(uintN),POINTER(JSBool)] + +JS_SetUCPropertyAttributes = libsmjs.JS_SetUCPropertyAttributes +JS_SetUCPropertyAttributes.restype = JSBool +JS_SetUCPropertyAttributes.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,c_uint,POINTER(JSBool)] + +JS_DefineUCPropertyWithTinyId = libsmjs.JS_DefineUCPropertyWithTinyId +JS_DefineUCPropertyWithTinyId.restype = JSBool +JS_DefineUCPropertyWithTinyId.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,c_char,c_long,CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,POINTER(jsval)),c_uint] + +JS_LookupUCProperty = libsmjs.JS_LookupUCProperty +JS_LookupUCProperty.restype = JSBool +JS_LookupUCProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,POINTER(jsval)] + +JS_GetUCProperty = libsmjs.JS_GetUCProperty +JS_GetUCProperty.restype = JSBool +JS_GetUCProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,POINTER(jsval)] + +JS_SetUCProperty = libsmjs.JS_SetUCProperty +JS_SetUCProperty.restype = JSBool +JS_SetUCProperty.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,POINTER(jsval)] + +JS_DeleteUCProperty2 = libsmjs.JS_DeleteUCProperty2 +JS_DeleteUCProperty2.restype = JSBool +JS_DeleteUCProperty2.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,POINTER(jsval)] + +JS_NewArrayObject = libsmjs.JS_NewArrayObject +JS_NewArrayObject.restype = POINTER(JSObject) +JS_NewArrayObject.argtypes = [POINTER(JSContext),c_int,POINTER(jsval)] + +JS_IsArrayObject = libsmjs.JS_IsArrayObject +JS_IsArrayObject.restype = JSBool +JS_IsArrayObject.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_GetArrayLength = libsmjs.JS_GetArrayLength +JS_GetArrayLength.restype = JSBool +JS_GetArrayLength.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jsuint)] + +JS_SetArrayLength = libsmjs.JS_SetArrayLength +JS_SetArrayLength.restype = JSBool +JS_SetArrayLength.argtypes = [POINTER(JSContext),POINTER(JSObject),c_uint] + +JS_HasArrayLength = libsmjs.JS_HasArrayLength +JS_HasArrayLength.restype = JSBool +JS_HasArrayLength.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jsuint)] + +JS_DefineElement = libsmjs.JS_DefineElement +JS_DefineElement.restype = JSBool +#JS_DefineElement.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int,c_long,JSPropertyOp,JSPropertyOp,c_uint] +JS_DefineElement.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int,c_long,c_void_p,c_void_p,c_uint] + +JS_AliasElement = libsmjs.JS_AliasElement +JS_AliasElement.restype = JSBool +JS_AliasElement.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_int] + +JS_LookupElement = libsmjs.JS_LookupElement +JS_LookupElement.restype = JSBool +JS_LookupElement.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int,POINTER(jsval)] + +JS_GetElement = libsmjs.JS_GetElement +JS_GetElement.restype = JSBool +JS_GetElement.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int,POINTER(jsval)] + +JS_SetElement = libsmjs.JS_SetElement +JS_SetElement.restype = JSBool +JS_SetElement.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int,POINTER(jsval)] + +JS_DeleteElement = libsmjs.JS_DeleteElement +JS_DeleteElement.restype = JSBool +JS_DeleteElement.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int] + +JS_DeleteElement2 = libsmjs.JS_DeleteElement2 +JS_DeleteElement2.restype = JSBool +JS_DeleteElement2.argtypes = [POINTER(JSContext),POINTER(JSObject),c_int,POINTER(jsval)] + +JS_ClearScope = libsmjs.JS_ClearScope +JS_ClearScope.restype = None +JS_ClearScope.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_Enumerate = libsmjs.JS_Enumerate +JS_Enumerate.restype = POINTER(JSIdArray) +JS_Enumerate.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_CheckAccess = libsmjs.JS_CheckAccess +JS_CheckAccess.restype = JSBool +JS_CheckAccess.argtypes = [POINTER(JSContext),POINTER(JSObject),c_long,c_int,POINTER(jsval),POINTER(uintN)] + +JS_SetCheckObjectAccessCallback = libsmjs.JS_SetCheckObjectAccessCallback +JS_SetCheckObjectAccessCallback.restype = JSCheckAccessOp +JS_SetCheckObjectAccessCallback.argtypes = [POINTER(JSRuntime),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_long,c_int,POINTER(jsval))] + +JS_GetReservedSlot = libsmjs.JS_GetReservedSlot +JS_GetReservedSlot.restype = JSBool +JS_GetReservedSlot.argtypes = [POINTER(JSContext),POINTER(JSObject),c_uint,POINTER(jsval)] + +JS_SetReservedSlot = libsmjs.JS_SetReservedSlot +JS_SetReservedSlot.restype = JSBool +JS_SetReservedSlot.argtypes = [POINTER(JSContext),POINTER(JSObject),c_uint,c_long] + +JS_SetPrincipalsTranscoder = libsmjs.JS_SetPrincipalsTranscoder +JS_SetPrincipalsTranscoder.restype = JSPrincipalsTranscoder +JS_SetPrincipalsTranscoder.argtypes = [POINTER(JSRuntime),CFUNCTYPE(JSBool,POINTER(JSXDRState),POINTER(POINTER(JSPrincipals)))] + +JS_SetObjectPrincipalsFinder = libsmjs.JS_SetObjectPrincipalsFinder +JS_SetObjectPrincipalsFinder.restype = JSObjectPrincipalsFinder +JS_SetObjectPrincipalsFinder.argtypes = [POINTER(JSContext),CFUNCTYPE(POINTER(JSPrincipals),POINTER(JSContext),POINTER(JSObject))] + +JS_NewFunction = libsmjs.JS_NewFunction +JS_NewFunction.restype = POINTER(JSFunction) +JS_NewFunction.argtypes = [POINTER(JSContext),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_uint,POINTER(jsval),POINTER(jsval)),c_uint,c_uint,POINTER(JSObject),c_char_p] + +JS_GetFunctionObject = libsmjs.JS_GetFunctionObject +JS_GetFunctionObject.restype = POINTER(JSObject) +JS_GetFunctionObject.argtypes = [POINTER(JSFunction)] + +JS_GetFunctionName = libsmjs.JS_GetFunctionName +JS_GetFunctionName.restype = c_char_p +JS_GetFunctionName.argtypes = [POINTER(JSFunction)] + +JS_GetFunctionId = libsmjs.JS_GetFunctionId +JS_GetFunctionId.restype = POINTER(JSString) +JS_GetFunctionId.argtypes = [POINTER(JSFunction)] + +JS_GetFunctionFlags = libsmjs.JS_GetFunctionFlags +JS_GetFunctionFlags.restype = uintN +JS_GetFunctionFlags.argtypes = [POINTER(JSFunction)] + +JS_ObjectIsFunction = libsmjs.JS_ObjectIsFunction +JS_ObjectIsFunction.restype = JSBool +JS_ObjectIsFunction.argtypes = [POINTER(JSContext),POINTER(JSObject)] + +JS_DefineFunctions = libsmjs.JS_DefineFunctions +JS_DefineFunctions.restype = JSBool +JS_DefineFunctions.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSFunctionSpec)] + +JS_DefineFunction = libsmjs.JS_DefineFunction +JS_DefineFunction.restype = POINTER(JSFunction) +JS_DefineFunction.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSObject),c_uint,POINTER(jsval),POINTER(jsval)),c_uint,c_uint] + +JS_CloneFunctionObject = libsmjs.JS_CloneFunctionObject +JS_CloneFunctionObject.restype = POINTER(JSObject) +JS_CloneFunctionObject.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSObject)] + +JS_BufferIsCompilableUnit = libsmjs.JS_BufferIsCompilableUnit +JS_BufferIsCompilableUnit.restype = JSBool +JS_BufferIsCompilableUnit.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_uint] + +JS_CompileScript = libsmjs.JS_CompileScript +JS_CompileScript.restype = POINTER(JSScript) +JS_CompileScript.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_uint,c_char_p,c_uint] + +JS_CompileScriptForPrincipals = libsmjs.JS_CompileScriptForPrincipals +JS_CompileScriptForPrincipals.restype = POINTER(JSScript) +JS_CompileScriptForPrincipals.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSPrincipals),c_char_p,c_uint,c_char_p,c_uint] + +JS_CompileUCScript = libsmjs.JS_CompileUCScript +JS_CompileUCScript.restype = POINTER(JSScript) +JS_CompileUCScript.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,c_char_p,c_uint] + +JS_CompileUCScriptForPrincipals = libsmjs.JS_CompileUCScriptForPrincipals +JS_CompileUCScriptForPrincipals.restype = POINTER(JSScript) +JS_CompileUCScriptForPrincipals.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSPrincipals),POINTER(jschar),c_uint,c_char_p,c_uint] + +JS_CompileFile = libsmjs.JS_CompileFile +JS_CompileFile.restype = POINTER(JSScript) +JS_CompileFile.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p] + +class _IO_marker(Structure): + _fields_ = [ + ('_next', c_void_p), + ('_sbuf', POINTER(_IO_FILE)), + ('_pos', c_int), + ] + +t__off_t = c_long + +_IO_lock_t = None + +t__quad_t = c_longlong + +t__off64_t = t__quad_t + +class _IO_FILE(Structure): + _fields_ = [ + ('_flags', c_int), + ('_IO_read_ptr', c_char_p), + ('_IO_read_end', c_char_p), + ('_IO_read_base', c_char_p), + ('_IO_write_base', c_char_p), + ('_IO_write_ptr', c_char_p), + ('_IO_write_end', c_char_p), + ('_IO_buf_base', c_char_p), + ('_IO_buf_end', c_char_p), + ('_IO_save_base', c_char_p), + ('_IO_backup_base', c_char_p), + ('_IO_save_end', c_char_p), + ('_markers', POINTER(_IO_marker)), + ('_chain', c_void_p), + ('_fileno', c_int), + ('_flags2', c_int), + ('_old_offset', t__off_t), + ('_cur_column', c_ushort), + ('_vtable_offset', c_char), + ('_shortbuf', c_char*1), + ('_lock', POINTER(_IO_lock_t)), + ('_offset', t__off64_t), + ('__pad1', c_void_p), + ('__pad2', c_void_p), + ('_mode', c_int), + ('_unused2', c_char*52), + ] + +FILE = _IO_FILE + +JS_CompileFileHandle = libsmjs.JS_CompileFileHandle +JS_CompileFileHandle.restype = POINTER(JSScript) +JS_CompileFileHandle.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(FILE)] + +JS_CompileFileHandleForPrincipals = libsmjs.JS_CompileFileHandleForPrincipals +JS_CompileFileHandleForPrincipals.restype = POINTER(JSScript) +JS_CompileFileHandleForPrincipals.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,POINTER(FILE),POINTER(JSPrincipals)] + +JS_NewScriptObject = libsmjs.JS_NewScriptObject +JS_NewScriptObject.restype = POINTER(JSObject) +JS_NewScriptObject.argtypes = [POINTER(JSContext),POINTER(JSScript)] + +JS_GetScriptObject = libsmjs.JS_GetScriptObject +JS_GetScriptObject.restype = POINTER(JSObject) +JS_GetScriptObject.argtypes = [POINTER(JSScript)] + +JS_DestroyScript = libsmjs.JS_DestroyScript +JS_DestroyScript.restype = None +JS_DestroyScript.argtypes = [POINTER(JSContext),POINTER(JSScript)] + +JS_CompileFunction = libsmjs.JS_CompileFunction +JS_CompileFunction.restype = POINTER(JSFunction) +JS_CompileFunction.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_uint,POINTER(c_char_p),c_char_p,c_uint,c_char_p,c_uint] + +JS_CompileFunctionForPrincipals = libsmjs.JS_CompileFunctionForPrincipals +JS_CompileFunctionForPrincipals.restype = POINTER(JSFunction) +JS_CompileFunctionForPrincipals.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSPrincipals),c_char_p,c_uint,POINTER(c_char_p),c_char_p,c_uint,c_char_p,c_uint] + +JS_CompileUCFunction = libsmjs.JS_CompileUCFunction +JS_CompileUCFunction.restype = POINTER(JSFunction) +JS_CompileUCFunction.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_uint,POINTER(c_char_p),POINTER(jschar),c_uint,c_char_p,c_uint] + +JS_CompileUCFunctionForPrincipals = libsmjs.JS_CompileUCFunctionForPrincipals +JS_CompileUCFunctionForPrincipals.restype = POINTER(JSFunction) +JS_CompileUCFunctionForPrincipals.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSPrincipals),c_char_p,c_uint,POINTER(c_char_p),POINTER(jschar),c_uint,c_char_p,c_uint] + +JS_DecompileScript = libsmjs.JS_DecompileScript +JS_DecompileScript.restype = POINTER(JSString) +JS_DecompileScript.argtypes = [POINTER(JSContext),POINTER(JSScript),c_char_p,c_uint] + +JS_DecompileFunction = libsmjs.JS_DecompileFunction +JS_DecompileFunction.restype = POINTER(JSString) +JS_DecompileFunction.argtypes = [POINTER(JSContext),POINTER(JSFunction),c_uint] + +JS_DecompileFunctionBody = libsmjs.JS_DecompileFunctionBody +JS_DecompileFunctionBody.restype = POINTER(JSString) +JS_DecompileFunctionBody.argtypes = [POINTER(JSContext),POINTER(JSFunction),c_uint] + +JS_ExecuteScript = libsmjs.JS_ExecuteScript +JS_ExecuteScript.restype = JSBool +JS_ExecuteScript.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSScript),POINTER(jsval)] + +# enumeration JSExecPart +JSEXEC_PROLOG = 0 +JSEXEC_MAIN = 1 + +JSExecPart = c_int + +JS_ExecuteScriptPart = libsmjs.JS_ExecuteScriptPart +JS_ExecuteScriptPart.restype = JSBool +JS_ExecuteScriptPart.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSScript),c_int,POINTER(jsval)] + +JS_EvaluateScript = libsmjs.JS_EvaluateScript +JS_EvaluateScript.restype = JSBool +JS_EvaluateScript.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_uint,c_char_p,c_uint,POINTER(jsval)] + +JS_EvaluateScriptForPrincipals = libsmjs.JS_EvaluateScriptForPrincipals +JS_EvaluateScriptForPrincipals.restype = JSBool +JS_EvaluateScriptForPrincipals.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSPrincipals),c_char_p,c_uint,c_char_p,c_uint,POINTER(jsval)] + +JS_EvaluateUCScript = libsmjs.JS_EvaluateUCScript +JS_EvaluateUCScript.restype = JSBool +JS_EvaluateUCScript.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(jschar),c_uint,c_char_p,c_uint,POINTER(jsval)] + +JS_EvaluateUCScriptForPrincipals = libsmjs.JS_EvaluateUCScriptForPrincipals +JS_EvaluateUCScriptForPrincipals.restype = JSBool +JS_EvaluateUCScriptForPrincipals.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSPrincipals),POINTER(jschar),c_uint,c_char_p,c_uint,POINTER(jsval)] + +JS_CallFunction = libsmjs.JS_CallFunction +JS_CallFunction.restype = JSBool +JS_CallFunction.argtypes = [POINTER(JSContext),POINTER(JSObject),POINTER(JSFunction),c_uint,POINTER(jsval),POINTER(jsval)] + +JS_CallFunctionName = libsmjs.JS_CallFunctionName +JS_CallFunctionName.restype = JSBool +JS_CallFunctionName.argtypes = [POINTER(JSContext),POINTER(JSObject),c_char_p,c_uint,POINTER(jsval),POINTER(jsval)] + +JS_CallFunctionValue = libsmjs.JS_CallFunctionValue +JS_CallFunctionValue.restype = JSBool +JS_CallFunctionValue.argtypes = [POINTER(JSContext),POINTER(JSObject),c_long,c_uint,POINTER(jsval),POINTER(jsval)] + +JS_SetBranchCallback = libsmjs.JS_SetBranchCallback +JS_SetBranchCallback.restype = JSBranchCallback +JS_SetBranchCallback.argtypes = [POINTER(JSContext),CFUNCTYPE(JSBool,POINTER(JSContext),POINTER(JSScript))] + +JS_IsRunning = libsmjs.JS_IsRunning +JS_IsRunning.restype = JSBool +JS_IsRunning.argtypes = [POINTER(JSContext)] + +JS_IsConstructing = libsmjs.JS_IsConstructing +JS_IsConstructing.restype = JSBool +JS_IsConstructing.argtypes = [POINTER(JSContext)] + +JS_IsAssigning = libsmjs.JS_IsAssigning +JS_IsAssigning.restype = JSBool +JS_IsAssigning.argtypes = [POINTER(JSContext)] + +JS_SetCallReturnValue2 = libsmjs.JS_SetCallReturnValue2 +JS_SetCallReturnValue2.restype = None +JS_SetCallReturnValue2.argtypes = [POINTER(JSContext),c_long] + +JS_NewString = libsmjs.JS_NewString +JS_NewString.restype = POINTER(JSString) +JS_NewString.argtypes = [POINTER(JSContext),c_char_p,c_uint] + +JS_NewStringCopyN = libsmjs.JS_NewStringCopyN +JS_NewStringCopyN.restype = POINTER(JSString) +JS_NewStringCopyN.argtypes = [POINTER(JSContext),c_char_p,c_uint] + +JS_NewStringCopyZ = libsmjs.JS_NewStringCopyZ +JS_NewStringCopyZ.restype = POINTER(JSString) +JS_NewStringCopyZ.argtypes = [POINTER(JSContext),c_char_p] + +JS_InternString = libsmjs.JS_InternString +JS_InternString.restype = POINTER(JSString) +JS_InternString.argtypes = [POINTER(JSContext),c_char_p] + +JS_NewUCString = libsmjs.JS_NewUCString +JS_NewUCString.restype = POINTER(JSString) +JS_NewUCString.argtypes = [POINTER(JSContext),POINTER(jschar),c_uint] + +JS_NewUCStringCopyN = libsmjs.JS_NewUCStringCopyN +JS_NewUCStringCopyN.restype = POINTER(JSString) +JS_NewUCStringCopyN.argtypes = [POINTER(JSContext),POINTER(jschar),c_uint] + +JS_NewUCStringCopyZ = libsmjs.JS_NewUCStringCopyZ +JS_NewUCStringCopyZ.restype = POINTER(JSString) +JS_NewUCStringCopyZ.argtypes = [POINTER(JSContext),POINTER(jschar)] + +JS_InternUCStringN = libsmjs.JS_InternUCStringN +JS_InternUCStringN.restype = POINTER(JSString) +JS_InternUCStringN.argtypes = [POINTER(JSContext),POINTER(jschar),c_uint] + +JS_InternUCString = libsmjs.JS_InternUCString +JS_InternUCString.restype = POINTER(JSString) +JS_InternUCString.argtypes = [POINTER(JSContext),POINTER(jschar)] + +JS_GetStringBytes = libsmjs.JS_GetStringBytes +JS_GetStringBytes.restype = c_char_p +JS_GetStringBytes.argtypes = [POINTER(JSString)] + +JS_GetStringChars = libsmjs.JS_GetStringChars +JS_GetStringChars.restype = POINTER(jschar) +JS_GetStringChars.argtypes = [POINTER(JSString)] + +JS_GetStringLength = libsmjs.JS_GetStringLength +JS_GetStringLength.restype = size_t +JS_GetStringLength.argtypes = [POINTER(JSString)] + +JS_CompareStrings = libsmjs.JS_CompareStrings +JS_CompareStrings.restype = intN +JS_CompareStrings.argtypes = [POINTER(JSString),POINTER(JSString)] + +JS_NewGrowableString = libsmjs.JS_NewGrowableString +JS_NewGrowableString.restype = POINTER(JSString) +JS_NewGrowableString.argtypes = [POINTER(JSContext),POINTER(jschar),c_uint] + +JS_NewDependentString = libsmjs.JS_NewDependentString +JS_NewDependentString.restype = POINTER(JSString) +JS_NewDependentString.argtypes = [POINTER(JSContext),POINTER(JSString),c_uint,c_uint] + +JS_ConcatStrings = libsmjs.JS_ConcatStrings +JS_ConcatStrings.restype = POINTER(JSString) +JS_ConcatStrings.argtypes = [POINTER(JSContext),POINTER(JSString),POINTER(JSString)] + +JS_UndependString = libsmjs.JS_UndependString +JS_UndependString.restype = POINTER(jschar) +JS_UndependString.argtypes = [POINTER(JSContext),POINTER(JSString)] + +JS_MakeStringImmutable = libsmjs.JS_MakeStringImmutable +JS_MakeStringImmutable.restype = JSBool +JS_MakeStringImmutable.argtypes = [POINTER(JSContext),POINTER(JSString)] + +JS_SetLocaleCallbacks = libsmjs.JS_SetLocaleCallbacks +JS_SetLocaleCallbacks.restype = None +JS_SetLocaleCallbacks.argtypes = [POINTER(JSContext),POINTER(JSLocaleCallbacks)] + +JS_GetLocaleCallbacks = libsmjs.JS_GetLocaleCallbacks +JS_GetLocaleCallbacks.restype = POINTER(JSLocaleCallbacks) +JS_GetLocaleCallbacks.argtypes = [POINTER(JSContext)] + +JS_ReportError = libsmjs.JS_ReportError +JS_ReportError.restype = None +JS_ReportError.argtypes = [POINTER(JSContext),c_char_p] + +JS_ReportErrorNumber = libsmjs.JS_ReportErrorNumber +JS_ReportErrorNumber.restype = None +JS_ReportErrorNumber.argtypes = [POINTER(JSContext),CFUNCTYPE(POINTER(JSErrorFormatString),c_void_p,c_char_p,c_uint),c_void_p,c_uint] + +JS_ReportErrorNumberUC = libsmjs.JS_ReportErrorNumberUC +JS_ReportErrorNumberUC.restype = None +JS_ReportErrorNumberUC.argtypes = [POINTER(JSContext),CFUNCTYPE(POINTER(JSErrorFormatString),c_void_p,c_char_p,c_uint),c_void_p,c_uint] + +JS_ReportWarning = libsmjs.JS_ReportWarning +JS_ReportWarning.restype = JSBool +JS_ReportWarning.argtypes = [POINTER(JSContext),c_char_p] + +JS_ReportErrorFlagsAndNumber = libsmjs.JS_ReportErrorFlagsAndNumber +JS_ReportErrorFlagsAndNumber.restype = JSBool +JS_ReportErrorFlagsAndNumber.argtypes = [POINTER(JSContext),c_uint,CFUNCTYPE(POINTER(JSErrorFormatString),c_void_p,c_char_p,c_uint),c_void_p,c_uint] + +JS_ReportErrorFlagsAndNumberUC = libsmjs.JS_ReportErrorFlagsAndNumberUC +JS_ReportErrorFlagsAndNumberUC.restype = JSBool +JS_ReportErrorFlagsAndNumberUC.argtypes = [POINTER(JSContext),c_uint,CFUNCTYPE(POINTER(JSErrorFormatString),c_void_p,c_char_p,c_uint),c_void_p,c_uint] + +JS_ReportOutOfMemory = libsmjs.JS_ReportOutOfMemory +JS_ReportOutOfMemory.restype = None +JS_ReportOutOfMemory.argtypes = [POINTER(JSContext)] + +JS_SetErrorReporter = libsmjs.JS_SetErrorReporter +JS_SetErrorReporter.restype = JSErrorReporter +JS_SetErrorReporter.argtypes = [POINTER(JSContext),CFUNCTYPE(None,POINTER(JSContext),c_char_p,POINTER(JSErrorReport))] + +JS_NewRegExpObject = libsmjs.JS_NewRegExpObject +JS_NewRegExpObject.restype = POINTER(JSObject) +JS_NewRegExpObject.argtypes = [POINTER(JSContext),c_char_p,c_uint,c_uint] + +JS_NewUCRegExpObject = libsmjs.JS_NewUCRegExpObject +JS_NewUCRegExpObject.restype = POINTER(JSObject) +JS_NewUCRegExpObject.argtypes = [POINTER(JSContext),POINTER(jschar),c_uint,c_uint] + +JS_SetRegExpInput = libsmjs.JS_SetRegExpInput +JS_SetRegExpInput.restype = None +JS_SetRegExpInput.argtypes = [POINTER(JSContext),POINTER(JSString),c_int] + +JS_ClearRegExpStatics = libsmjs.JS_ClearRegExpStatics +JS_ClearRegExpStatics.restype = None +JS_ClearRegExpStatics.argtypes = [POINTER(JSContext)] + +JS_ClearRegExpRoots = libsmjs.JS_ClearRegExpRoots +JS_ClearRegExpRoots.restype = None +JS_ClearRegExpRoots.argtypes = [POINTER(JSContext)] + +JS_IsExceptionPending = libsmjs.JS_IsExceptionPending +JS_IsExceptionPending.restype = JSBool +JS_IsExceptionPending.argtypes = [POINTER(JSContext)] + +JS_GetPendingException = libsmjs.JS_GetPendingException +JS_GetPendingException.restype = JSBool +JS_GetPendingException.argtypes = [POINTER(JSContext),POINTER(jsval)] + +JS_SetPendingException = libsmjs.JS_SetPendingException +JS_SetPendingException.restype = None +JS_SetPendingException.argtypes = [POINTER(JSContext),c_long] + +JS_ClearPendingException = libsmjs.JS_ClearPendingException +JS_ClearPendingException.restype = None +JS_ClearPendingException.argtypes = [POINTER(JSContext)] + +JS_SaveExceptionState = libsmjs.JS_SaveExceptionState +JS_SaveExceptionState.restype = POINTER(JSExceptionState) +JS_SaveExceptionState.argtypes = [POINTER(JSContext)] + +JS_RestoreExceptionState = libsmjs.JS_RestoreExceptionState +JS_RestoreExceptionState.restype = None +JS_RestoreExceptionState.argtypes = [POINTER(JSContext),POINTER(JSExceptionState)] + +JS_DropExceptionState = libsmjs.JS_DropExceptionState +JS_DropExceptionState.restype = None +JS_DropExceptionState.argtypes = [POINTER(JSContext),POINTER(JSExceptionState)] + +JS_ErrorFromException = libsmjs.JS_ErrorFromException +JS_ErrorFromException.restype = POINTER(JSErrorReport) +JS_ErrorFromException.argtypes = [POINTER(JSContext),c_long] + +JSACC_TYPEMASK = (JSACC_WRITE - 1) +JSLL_MAXINT = JSLL_MaxInt() +JSLL_MININT = JSLL_MinInt() +JSLL_ZERO = JSLL_Zero() + +_tabs = 0 + +def _log_call(func, *args, **kargs): + global _tabs + _tabs += 1 + try: + print '\t'*_tabs + 'CALL {',func,args,kargs,'}' + r = func(*args,**kargs) + print '\t'*_tabs + "RETURNED",r + finally: + _tabs += 0 + return r + +def log_call(func): + return lambda *args, **kargs: _log_call(func,*args,**kargs) + +class JSError(Exception): + pass + +def dict_from_JShash(context, hash): + #~ cdef JSIdArray *prop_arr + #~ cdef int i + #~ cdef jsval jskey + #~ cdef jsval jsvalue + #~ cdef JSObject *obj + + + cx = context.cx + + prop_arr = JS_Enumerate(cx, hash).contents + + jskey = jsval() + jsvalue = jsval() + + d = {} + + for i in range(prop_arr.length): + vector = cast(addressof(prop_arr.vector),POINTER(c_long)) + JS_IdToValue(cx, (vector)[i], byref(jskey)) + + if JSVAL_IS_STRING(jskey): + key = JS_GetStringBytes(JSVAL_TO_STRING(jskey)) + JS_GetProperty(cx, hash, key, byref(jsvalue)) + elif JSVAL_IS_INT(jskey): + key = JSVAL_TO_INT(jskey) + JS_GetElement(cx, hash, key, byref(jsvalue)) + else: + assert False, "can't happen" + + if JSVAL_IS_PRIMITIVE(jsvalue): + d[key] = Py_from_JSprimitive(jsvalue) + else: + if JSVAL_IS_OBJECT(jsvalue): + obj = JSVAL_TO_OBJECT(jsvalue) + if JS_IsArrayObject(cx, obj): + d[key] = list_from_JSarray(context, obj) + else: + d[key] = dict_from_JShash(context, obj) + + JS_DestroyIdArray(cx, prop_arr) + + return d + +def list_from_JSarray(context, array): + #~ cdef int nr_elems, i + #~ cdef jsval elem + #~ cdef JSObject *jsobj + + nr_elems = c_uint() + elem = jsval() + + cx = context.cx + JS_GetArrayLength(cx, array, byref(nr_elems)) + nr_elems = nr_elems.value + + l = [None]*nr_elems + + for i in range(nr_elems): + JS_GetElement(cx, array, i, byref(elem)) + + if JSVAL_IS_PRIMITIVE(elem): + l[i] = Py_from_JSprimitive(elem) + elif JSVAL_IS_OBJECT(elem): + jsobj = JSVAL_TO_OBJECT(elem) + if JS_IsArrayObject(cx, jsobj): + l[i] = list_from_JSarray(context, jsobj) + else: + l[i] = dict_from_JShash(context, jsobj) + + return l + +def Py_from_JSprimitive(v): + # JS_NULL is null, JS_VOID is undefined + if JSVAL_IS_NULL(v) or JSVAL_IS_VOID(v): + return None + elif JSVAL_IS_INT(v): + return JSVAL_TO_INT(v) + elif JSVAL_IS_DOUBLE(v): + return JSVAL_TO_DOUBLE(v)[0] + elif JSVAL_IS_STRING(v): + return JS_GetStringBytes(JSVAL_TO_STRING(v)) + elif JSVAL_IS_BOOLEAN(v): + return bool(JSVAL_TO_BOOLEAN(v)) + else: + raise SystemError("unknown primitive type") + +def Py_from_JS(context, v): + # Convert JavaScript value to equivalent Python value. + #~ cdef JSObject *object + #~ cdef ProxyObject proxy_obj + #~ cdef Context context + + if JSVAL_IS_PRIMITIVE(v): + return Py_from_JSprimitive(v) + else: + if JSVAL_IS_OBJECT(v): + object = JSVAL_TO_OBJECT(v) + + if JS_IsArrayObject(context.cx, object): + return list_from_JSarray(context, object) + else: + try: + proxy_obj = context.get_object(object) + except ValueError: + return dict_from_JShash(context, object) + else: + return proxy_obj.obj + +def isinteger(x): + return bool(compat_isinstance(x, (IntType, LongType))) + +def isfloat(x): + return bool(compat_isinstance(x, FloatType)) + +def isstringlike(x): + return bool(compat_isinstance(x, StringTypes)) + +def issequence(x): + return bool(compat_isinstance(x, (ListType, TupleType))) + +def ismapping(x): + return bool(compat_isinstance(x, DictType)) + +def js_classname(pobj): + # class or instance? + try: + klass = pobj.__class__ + except AttributeError: + klass = pobj + + try: + name = klass.js_name + except AttributeError: + name = klass.__name__ + + if not isstringlike(name): + raise AttributeError("%s js_name attribute is not string-like" % klass) + return name + +def JS_from_Py(context, parent, py_obj): + # Convert Python value to equivalent JavaScript value. + #~ cdef JSObject *jsobj + #~ cdef JSString *s + + #~ cdef jsval elem + #~ cdef jsval rval + + #~ cdef int i + #~ cdef int nr_elems + #~ cdef jsval *elems + #~ cdef JSObject *arr_obj + + #~ cdef Context context + #~ cdef ProxyClass proxy_class + #~ cdef ProxyObject proxy_obj + + rval = jsval() + cx = context.cx + + if py_obj is None: + return JSVAL_VOID + elif isinteger(py_obj): + return INT_TO_JSVAL(py_obj) + elif isfloat(py_obj): + d = JS_NewDouble(cx, py_obj) + if d == None: + raise SystemError("can't create new double") + return DOUBLE_TO_JSVAL(d) + elif isstringlike(py_obj): + s = JS_NewStringCopyN(cx, py_obj, len(py_obj)) + if s == None: + raise SystemError("can't create new string") + return STRING_TO_JSVAL(s) + elif ismapping(py_obj): + jsobj = JS_NewObject(cx, None, None, None) + + if jsobj == None: + raise SystemError("can't create new object") + else: + # assign properties + for key, value in py_obj.iteritems(): + elem = JS_from_Py(context, parent, value) + ok = JS_DefineProperty(cx, jsobj, key, elem, None, None, JSPROP_ENUMERATE) + if not ok: + raise SystemError("can't define property") + + return OBJECT_TO_JSVAL(jsobj) + + elif issequence(py_obj): + arr_obj = JS_NewArrayObject(cx, 0, None) + + if arr_obj == None: + raise SystemError("can't create new array object") + else: + for i in range(len(py_obj)): + elem = JS_from_Py(context, parent, py_obj[i]) + ok = JS_DefineElement(cx, arr_obj, i, elem, None, None, JSPROP_ENUMERATE) + if not ok: + raise SystemError("can't define element") + + return OBJECT_TO_JSVAL(arr_obj) + elif compat_isinstance(py_obj, MethodType): + # XXX leak? -- calls JS_DefineFunction every time a method + # is called + return context.new_method(py_obj) + elif compat_isinstance(py_obj, (FunctionType, LambdaType)): + # XXX implement me? + return JSVAL_VOID + else: + # If we get here, py_obj is probably a Python class or a class instance. + # XXXX could problems be caused if some weird object such a module or + # regexp or code object were, eg., returned by a Python function? + + # is object already bound to a JS proxy?... + try: + proxy_obj = context.get_object_from_py(py_obj) + except ValueError: + # ...no, so create and register a new JS proxy + proxy_class = context.get_class(js_classname(py_obj)) + # XXXX I have *no idea* if using globj as the proto here is + # correct! If I don't put globj in here, JS does a get_property + # on an object that hasn't been registered with the Context -- + # dunno why. + jsobj = proxy_class.new_object(parent=parent,py_obj = py_obj) # JS_NewObject(cx, proxy_class.jsc, context.globj, parent) + if jsobj == None: + raise SystemError("couldn't look up or create new object") + else: + # ...yes, use existing proxy + jsobj = proxy_obj.jsobj + return OBJECT_TO_JSVAL(jsobj) + +class Runtime: + + def __init__(self, memsize = (8*1024*1024), max_context = 32): + + self.rt = JS_Init(memsize) + self._cxs = [] + self._max_context = max_context + + def __dealloc__(self): + JS_Finish(self.rt) + + def new_context(self, globj = None, stacksize = 8192): + context = Context(globj, self, stacksize) + self._cxs.append(context) + if len(self._cxs) > self._max_context: + old_cx = self._cxs.pop(0) + old_cx.alive = False + JS_DestroyContext(old_cx.cx) + context.initialize() + return context + +import sys, inspect + +class ProxyObject: + def __init__(self, obj, jsobj = None): + self.obj = obj + self.jsobj = jsobj + +def compat_isinstance(obj, tuple_or_obj): + if type(tuple_or_obj) == TupleType: + for otherobj in tuple_or_obj: + if isinstance(obj, otherobj): + return True + return False + return isinstance(obj, tuple_or_obj) + +from types import * + +class ProxyClass: + def __init__(self, context, classobject, bind_constructor, is_global, flags): + self.context = context + name = js_classname(classobject) + self.classobject = classobject + self.jsclass = JSClass() + self.jsclass.name = name + self.jsclass.flags = flags + self.jsclass.addProperty = JS_PropertyStub + self.jsclass.delProperty = JS_PropertyStub + self.jsclass.getProperty = self.context._get_property + self.jsclass.setProperty = self.context._set_property + self.jsclass.enumerate = JS_EnumerateStub + if is_global: + self._resolve_global = JSResolveOp(self.resolve_global) + self.jsclass.resolve = self._resolve_global + else: + self.jsclass.resolve = JS_ResolveStub + self.jsclass.convert = JS_ConvertStub + self.jsclass.finalize = JS_FinalizeStub + + if bind_constructor: + self._constructor_cb = JSNative(self.constructor_cb) + res = JS_InitClass(context.cx, context._global, None, self.jsclass, self._constructor_cb, 0, None, None, None, None) + assert res, "couldn't initialise JavaScript proxy class" + + def resolve_global(self, cx, obj, id): + #~ cdef Context context + #~ cdef ProxyObject proxy_obj + #~ cdef object thing + #~ cdef object key + + try: + proxy_obj = self.context.get_object(obj) + thing = proxy_obj.obj + key = Py_from_JS(self.context, id) + if type(key) == str and hasattr(thing, key): + attr = getattr(thing, key) + if compat_isinstance(attr, MethodType): + self.context.bind_callable(key, attr) + else: + self.context.bind_attribute(key, thing, key) + return JS_TRUE + except: + return self.context.report() + return JS_FALSE + + def constructor_cb(self, cx, obj, argc, argv, rval): + #~ cdef ProxyClass proxy_class + #~ cdef Context context + #~ cdef char *fname + #~ cdef JSFunction *func + #~ cdef int arg + #~ cdef ProxyObject object + func = JS_ValueToFunction(cx, argv[-2]) + if not func: + msg = "couldn't get JS constructor" + JS_ReportError(cx, msg) + return JS_FALSE + fname = JS_GetFunctionName(func) + try: + args = [] + for arg in range(argc): + args.append(Py_from_JS(cx, argv[arg])) + if hasattr(self.classobject, "js_constructor"): + py_rval = self.classobject.js_constructor[0](self.context, *args) + else: + py_rval = self.classobject(*args) + self.context.register_object(py_rval, obj) + except: + self.context.report() + + return JS_TRUE + + def new_object(self, parent = None, py_obj = None): + proto = self.context._global + jsobj = JS_NewObject(self.context.cx, byref(self.jsclass), proto, parent) + if py_obj: + self.context.register_object(py_obj, jsobj) + return jsobj + +stderr = sys.stderr + +class DestroyedContext(Exception): + def __str__(self): + return "Context Memory has been released previously by Runtime Class sanity checks." + +class Context: + cx = None + + class global_class: + js_name = 'global' + + _global = None + + def __init__(self, globj, runtime, stacksize): + self.cx = JS_NewContext(runtime.rt, stacksize) + self.alive = True + self.classes = [] + self.objects = [] + self.funcs = [] + #self.runtime = runtime + self._get_property = JSPropertyOp(self.get_property) + self._set_property = JSPropertyOp(self.set_property) + self._bound_method_cb = JSNative(self.bound_method_cb) + if not globj: + globj = self.global_class() + global_class = self.bind_class(globj.__class__, bind_constructor = False, is_global = True) + self._global = global_class.new_object(py_obj = globj) + assert self._global + + def get_object_from_py(self, py_obj): + for proxy_obj in self.objects: + if proxy_obj.obj == py_obj: + return proxy_obj + raise ValueError, py_obj + + def get_object(self, jsobj): + for proxy_obj in self.objects: + if addressof(proxy_obj.jsobj.contents) == addressof(jsobj.contents): + return proxy_obj + raise ValueError, jsobj.contents + + def get_class(self, name): + #cdef ProxyClass cl + for cl in self.classes: + if cl.jsclass.name == name: + return cl + raise ValueError("no class named '%s' is bound" % name) + + def get_global(self, name): + """Only works with undotted names. Use eval_script for anything + else.""" + # XXXX probably best to get rid of this -- eval_script does the job + #cdef jsval jsv + if not isstringlike(name): + raise TypeError("name must be string-like") + jsv = jsval() + if not JS_GetProperty(self.cx, self._global, name, byref(jsv)): + raise SystemError("can't get JavaScript property") + val = Py_from_JS(self, jsv) + if val is None: + raise ValueError("no global named '%s'" % name) + ## # XXXX why does this hang?? + ## raise AttributeError("%s instance has no attribute '%s'" % + ## (self.__class__.__name__, name)) + else: + return val + + def register_object(self, obj, jsobj): + self.objects.append(ProxyObject(obj, jsobj)) + + def report(self): + import traceback + self.report_error(traceback.format_exc()) + + def report_error(self, msg, *args): + JS_ReportError(self.cx, msg, *args) + + def error_reporter(self, cx, text, er): + print >> stderr, text + + #def __del__(self): + #JS_DestroyContext(self.cx) + + def new_method(self, py_method): + #~ cdef JSFunction *method + #~ cdef JSObject *method_obj + + # Method (hence its func_name attribute) will go away only when the + # context is GC'd by Python, because the Python function object is + # stored in Context extension type instance. XXX erm, no it isn't: + # the class instance is kept in the ProxyClass, but the instance's + # function is not. Hmm... XXX also, JS objects exist in runtime, + # not in context! + method = JS_NewFunction(self.cx, self._bound_method_cb, 0, 0, None, py_method.func_name) + method_obj = JS_GetFunctionObject(method) + return OBJECT_TO_JSVAL(method_obj) + + def get_property(self, cx, obj, id, vp): + #~ cdef Context context + #~ cdef ProxyObject proxy_obj + #~ cdef object thing + #~ cdef object key + + try: + key = Py_from_JS(self, id) + proxy_obj = self.get_object(obj) + thing = proxy_obj.obj + if type(key) == IntType: + try: + attr = thing[key] + except: + pass + else: + vp[0] = JS_from_Py(self, obj, attr) + elif type(key) == StringType: + if not key.startswith("_"): + try: + attr = getattr(thing, key) + except: + pass + else: + vp[0] = JS_from_Py(self, obj, attr) + else: + #in original version: assert False + return JS_FALSE + return JS_TRUE + except: + self.report() + return JS_FALSE + + def set_property(self, cx, obj, id, vp): + #~ cdef Context context + #~ cdef ProxyObject proxy_obj + #~ cdef object thing + #~ cdef object key + #~ cdef object value + #~ cdef object attr + + try: + proxy_obj = self.get_object(obj) + + thing = proxy_obj.obj + key = Py_from_JS(self, id) + value = Py_from_JS(self, vp[0]) + if type(key) == IntType: + try: + thing[key] = value + except: + pass + elif type(key) == StringType: + if hasattr(thing, key) and not key.startswith("_"): + attr = getattr(thing, key) + if not callable(attr): + try: + setattr(thing, key, value) + except: + pass + else: + assert False + return JS_TRUE + except: + return self.report() + + return JS_FALSE + + def bound_method_cb(self, cx, obj, argc, argv, rval): + #~ cdef Context context + #~ cdef JSFunction *func + #~ cdef int arg + #~ cdef JSClass *jsclass + #~ cdef ProxyObject proxy_obj + + func = JS_ValueToFunction(cx, argv[-2]) + method_name = JS_GetFunctionName(func) + jsclass = JS_GetClass(self.cx, obj) + + try: + proxy_obj = self.get_object(obj) + + py_args = [] + for arg in range(argc): + py_args.append(Py_from_JS(self, argv[arg])) + meth = getattr(proxy_obj.obj, method_name) + py_rval = meth(*py_args) + rval[0] = JS_from_Py(self, obj, py_rval) + except: + return self.report() + return JS_TRUE + + def get_callback_fn(self, name): + for cbname, fn in self.funcs: + if cbname == name: + return fn + raise ValueError("no callback function named '%s' is bound" % name) + + def function_cb(self, cx, obj, argc, argv, rval): + #~ cdef JSFunction *jsfunc + #~ cdef int i + #~ cdef char *name + + # XXX is this argv[-2] documented anywhere?? + jsfunc = JS_ValueToFunction(cx, argv[-2]) + if not jsfunc: + return JS_FALSE + name = JS_GetFunctionName(jsfunc) + if not name: + return JS_FALSE + + try: + callback = self.get_callback_fn(name) + args = [] + for i in range(argc): + args.append(Py_from_JS(self, argv[i])) + pyrval = callback(*args) + + # XXX shouldn't NULL be the JSObject if it's a Python class instance? + rval[0] = JS_from_Py(self, None, pyrval) + except: + return self.report() + + return JS_TRUE + + def bind_callable(self, name, function): + if not callable(function): + raise ValueError("not a callable object") + self._function_cb = JSNative(self.function_cb) + self.funcs.append((name, function)) + JS_DefineFunction(self.cx, self._global, name, self._function_cb, 0, 0) + + def bind_attribute(self, name, obj, attr_name): + #cdef jsval initial_value + if not isstringlike(name): + raise TypeError("name must be string-like") + if not isstringlike(attr_name): + raise TypeError("name must be string-like") + initial_value = JS_from_Py(self, None, getattr(obj, name)) + JS_DefineProperty(self.cx, self._global, name, initial_value, cast(self._get_property,c_void_p), cast(self._set_property,c_void_p), 0) + + def bind_class(self, classobject, bind_constructor = True, is_global = False, flags = 0): + if not inspect.isclass(classobject): raise TypeError("klass must be a class") + if not isinteger(flags): raise TypeError("flags must be an integer") + + proxy_class = ProxyClass(self, classobject, bind_constructor, is_global, flags) + self.classes.append(proxy_class) + return proxy_class + + def bind_object(self, name, obj): + #~ cdef JSObject *jsobj + #~ cdef ProxyClass proxy_class + + if not isstringlike(name): + raise TypeError("name must be string-like") + + proxy_class = self.get_class(js_classname(obj)) + jsobj = JS_DefineObject(self.cx, self._global, name, proxy_class.jsclass, None, JSPROP_READONLY) + if jsobj: + self.register_object(obj, jsobj) + else: + raise ValueError("failed to bind Python object %s" % obj) + + def initialize(self): + JS_InitStandardClasses(self.cx, self._global) + self._error_reporter = JSErrorReporter(self.error_reporter) + JS_SetErrorReporter(self.cx, self._error_reporter) + + def eval_script(self, script): + if not isstringlike(script): + raise TypeError("name must be string-like") + if not self.alive: + raise DestroyedContext + rval = jsval() + if not JS_EvaluateScript(self.cx, self._global, script, len(script), 'Python', 0, byref(rval)): + raise JSError, "Failed to execute script." + + retval = Py_from_JS(self, rval) + JS_MaybeGC(self.cx) + return retval + +if __name__ == '__main__': + rt = Runtime() + cx = rt.new_context() + + class foo: + def test(self): + return 1.0 + + cx.bind_class(foo, bind_constructor=True) + f = cx.eval_script("""var f = new foo(); + f.test(); // script return value + """) + print f diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py new file mode 100644 index 000000000..069d22d19 --- /dev/null +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -0,0 +1,68 @@ +""" +LinkExtractor provides en efficient way to extract links from pages +""" + +from scrapy.utils.python import FixedSGMLParser +from scrapy.utils.url import urljoin_rfc as urljoin + +class LinkExtractor(FixedSGMLParser): + """LinkExtractor are used to extract links from web pages. They are + instantiated and later "applied" to a Response using the extract_urls + method which must receive a Response object and return a dict whoose keys + are the (absolute) urls to follow, and its values any arbitrary data. In + this case the values are the text of the hyperlink. + + This is the base LinkExtractor class that provides enough basic + functionality for extracting links to follow, but you could override this + class or create a new one if you need some additional functionality. The + only requisite is that the new (or overrided) class must provide a + extract_urls method that receives a Response and returns a dict with the + links to follow as its keys. + + The constructor arguments are: + + * tag (string or function) + * a tag name which is used to search for links (defaults to "a") + * a function which receives a tag name and returns whether to scan it + * attr (string or function) + * an attribute name which is used to search for links (defaults to "href") + * a function which receives an attribute name and returns whether to scan it + """ + + def __init__(self, tag="a", attr="href"): + FixedSGMLParser.__init__(self) + self.scan_tag = tag if callable(tag) else lambda t: t == tag + self.scan_attr = attr if callable(attr) else lambda a: a == attr + self.inside_link = False + + def extract_urls(self, response): + self.reset() + self.feed(response.body.to_string()) + self.close() + + base_url = self.base_url if self.base_url else response.url + urls = {} + for link, text in self.links.iteritems(): + urls[urljoin(base_url, link)] = text + return urls + + def reset(self): + FixedSGMLParser.reset(self) + self.links = {} + self.base_url = None + + def unknown_starttag(self, tag, attrs): + if tag == 'base': + self.base_url = dict(attrs).get('href') + if self.scan_tag(tag): + for attr, value in attrs: + if self.scan_attr(attr): + self.links[value] = "" + self.inside_link = value + + def unknown_endtag(self, tag): + self.inside_link = False + + def handle_data(self, data): + if self.inside_link and not self.links.get(self.inside_link, None): + self.links[self.inside_link] = data diff --git a/scrapy/trunk/scrapy/management/__init__.py b/scrapy/trunk/scrapy/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/management/telnet.py b/scrapy/trunk/scrapy/management/telnet.py new file mode 100644 index 000000000..52178785c --- /dev/null +++ b/scrapy/trunk/scrapy/management/telnet.py @@ -0,0 +1,42 @@ +import pprint + +from twisted.internet import reactor +from twisted.conch import manhole, telnet +from twisted.conch.insults import insults +from twisted.internet import protocol + +from scrapy.extension import extensions +from scrapy.core.manager import scrapymanager +from scrapy.core.engine import scrapyengine +from scrapy.stats import stats +from scrapy.conf import settings + +try: + import guppy + hpy = guppy.hpy() +except ImportError: + hpy = None + +telnet_namespace = {'ee': scrapyengine, + 'em': scrapymanager, + 'ex': extensions.enabled, + 'st': stats, + 'h': hpy, + 'p': pprint.pprint} # useful shortcut for debugging + +def makeProtocol(): + return telnet.TelnetTransport(telnet.TelnetBootstrapProtocol, + insults.ServerProtocol, + manhole.Manhole, telnet_namespace) + +class TelnetConsole(protocol.ServerFactory): + + def __init__(self): + if not settings.getbool('TELNETCONSOLE_ENABLED'): + return + + #protocol.ServerFactory.__init__(self) + self.protocol = makeProtocol + self.noisy = False + port = settings.getint('TELNETCONSOLE_PORT') + scrapyengine.listenTCP(port, self) diff --git a/scrapy/trunk/scrapy/management/web.py b/scrapy/trunk/scrapy/management/web.py new file mode 100644 index 000000000..f9d828546 --- /dev/null +++ b/scrapy/trunk/scrapy/management/web.py @@ -0,0 +1,69 @@ +import re +import socket +from datetime import datetime + +from twisted.internet import reactor +from twisted.web import server, resource +from pydispatch import dispatcher + +from scrapy.core.engine import scrapyengine +from scrapy.conf import settings + +# web management signals +webconsole_discover_module = object() + +urlpath_re = re.compile(r"^/(\w+)/") + +def banner(module=None): + s = "\n" + s += "Scrapy\n" + s += "\n" + s += "

    Scrapy web console

    \n" + now = datetime.now() + uptime = now - scrapyengine.start_time + s += "

    Bot: %s | Host: %s | Uptime: %s | Time: %s

    \n" % (settings['BOT_NAME'], socket.gethostname(), str(uptime), str(now)) + if module: + s += "

    %s

    \n" % (module.webconsole_id, module.webconsole_name) + return s + +class WebConsoleResource(resource.Resource): + isLeaf = True + + @property + def modules(self): + if not hasattr(self, '_modules'): + self._modules = {} + for _, obj in dispatcher.send(signal=webconsole_discover_module, sender=self.__class__): + self._modules[obj.webconsole_id] = obj + return self._modules + + def render_GET(self, request): + m = urlpath_re.search(request.path) + if m: + return self.modules[m.group(1)].webconsole_render(request) + else: + return self.module_list() + + render_POST = render_GET + + def module_list(self): + s = banner() + s += "

    Available modules:

    \n" + s += "
      \n" + for name, obj in self.modules.iteritems(): + s += "
    • %s
      • \n" % (name, obj.webconsole_name) + s += "
    \n" + s += "\n" + s += "\n" + return s + +class WebConsole(server.Site): + + def __init__(self): + if not settings.getbool('WEBCONSOLE_ENABLED'): + return + + server.Site.__init__(self, WebConsoleResource()) + self.noisy = False + port = settings.getint('WEBCONSOLE_PORT') + scrapyengine.listenTCP(port, self) diff --git a/scrapy/trunk/scrapy/patches/__init__.py b/scrapy/trunk/scrapy/patches/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/patches/monkeypatches.py b/scrapy/trunk/scrapy/patches/monkeypatches.py new file mode 100644 index 000000000..bad1edebd --- /dev/null +++ b/scrapy/trunk/scrapy/patches/monkeypatches.py @@ -0,0 +1,45 @@ +""" +Monkey patches + +These are generally a bad idea. +""" +from twisted.web.client import HTTPClientFactory + + +# Extend limit for BeautifulSoup parsing loops +#import sys +#sys.setrecursionlimit(7400) + +def apply_patches(): + patch_HTTPClientFactory_gotHeaders() + + +# XXX: Monkeypatch for twisted.web.client-HTTPClientFactory +# HTTPClientFactory.gotHeaders dies when parsing malformed cookies, +# and the crawler is getting malformed cookies from this site. +_old_gotHeaders = HTTPClientFactory.gotHeaders +# Cookies format: http://www.ietf.org/rfc/rfc2109.txt +# I have choosen not to filter based on this, so we don't filter invalid +# values that could be managed correctly by twisted. +#_COOKIES = re.compile(r'^[^=;]+=[^=;]*(;[^=;]+=[^=;])*$') + +def _is_good_cookie(cookie): + """ Check if a given cookie would make gotHeaders raise an exception """ + cookparts = cookie.split(';') + cook = cookparts[0] + return len(cook.split('=', 1)) == 2 + +def _new_gotHeaders(self, headers): + """ Remove cookies that would make twisted raise an exception """ + if headers.has_key('set-cookie'): + cookies = [cookie for cookie in headers['set-cookie'] + if _is_good_cookie(cookie)] + if cookies: + headers['set-cookie'] = cookies + else: + del headers['set-cookie'] + + return _old_gotHeaders(self, headers) + +def patch_HTTPClientFactory_gotHeaders(): + setattr(HTTPClientFactory, 'gotHeaders', _new_gotHeaders) diff --git a/scrapy/trunk/scrapy/replay/__init__.py b/scrapy/trunk/scrapy/replay/__init__.py new file mode 100644 index 000000000..e2d491f5b --- /dev/null +++ b/scrapy/trunk/scrapy/replay/__init__.py @@ -0,0 +1,177 @@ +import os +import shelve +import shutil +import tempfile +import tarfile + +from pydispatch import dispatcher + +from scrapy.core import signals, log +from scrapy.core.manager import scrapymanager +from scrapy.core.exceptions import NotConfigured +from scrapy.conf import settings + +class Replay(object): + """ + This is the Replay extension, used to replay a crawling and see if the + items scraped are the same. + """ + + def __init__(self, repfile, mode='play', usedir=False): + """ + Available modes: record, play, update + + repfile can be either a dir (is usedir=True) or a tar.gz file (if + usedir=False) + """ + + # XXX: this is ugly, and should be removed. but how? + cachemw = 'scrapy.contrib.downloadermiddleware.cache.CacheMiddleware' + if not cachemw in settings['DOWNLOADER_MIDDLEWARES']: + raise NotConfigured("Cache middleware must be enabled to use Replay") + + self.recording = mode == 'record' + self.updating = mode == 'update' + self.playing = mode == 'play' + + self.options = {} + self.scraped_old = {} + self.scraped_new = {} + self.passed_old = {} + self.passed_new = {} + self.responses_old = {} + self.responses_new = {} + + self._opendb(repfile, usedir) + + settings.overrides['CACHE2_DIR'] = self.cache2dir + settings.overrides['CACHE2_IGNORE_MISSING'] = self.playing or self.updating + settings.overrides['CACHE2_SECTORIZE'] = False + + dispatcher.connect(self.engine_started, signal=signals.engine_started) + dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) + dispatcher.connect(self.item_scraped, signal=signals.item_scraped) + dispatcher.connect(self.item_passed, signal=signals.item_passed) + dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) + + def play(self, args=None, opts=None): + if self.recording: + raise ValueError("Replay.play() not available in record mode") + + args = args or self.options['args'] + opts = opts or self.options['opts'] + scrapymanager.runonce(*args, **opts) + + def record(self, args=None, opts=None): + self.options.clear() + self.options['args'] = args or [] + self.options['opts'] = opts or {} + + if os.path.exists(self.cache2dir): + shutil.rmtree(self.cache2dir) + else: + os.mkdir(self.cache2dir) + + def update(self, args=None, opts=None): + self.updating = True + + self.play(args, opts) + + def engine_started(self): + log.msg("Replay: recording session in %s" % self.repfile) + + def engine_stopped(self): + if self.recording or self.updating: + log.msg("Replay: recorded in %s: %d/%d scraped/passed items, %d downloaded responses" % \ + (self.repfile, len(self.scraped_old), len(self.passed_old), len(self.responses_old))) + self._closedb() + self.cleanup() + + def item_scraped(self, item, spider): + if self.recording or self.updating: + self.scraped_old[str(item.guid)] = item.copy() + else: + self.scraped_new[str(item.guid)] = item.copy() + + def item_passed(self, item, spider): + if self.recording or self.updating: + self.passed_old[str(item.guid)] = item.copy() + else: + self.passed_new[str(item.guid)] = item.copy() + + def response_downloaded(self, response, spider): + #key = response.request.fingerprint() + key = response.version() + if self.recording and key: + self.responses_old[key] = response.copy() + elif key: + self.responses_new[key] = response.copy() + + def _opendb(self, repfile, usedir): + if usedir: + if os.path.isdir(repfile): + replay_dir = repfile + elif self.recording: + os.makedirs(repfile) + replay_dir = repfile + else: + raise IOError("No such dir: " % repfile) + elif os.path.exists(repfile) and (self.playing or self.updating): + if tarfile.is_tarfile(repfile): + replay_dir = tempfile.mkdtemp(prefix="replay-") + tar = tarfile.open(repfile) + tar.extractall(replay_dir) + tar.close() + else: + raise IOError("Wrong tarfile: %s" % repfile) + elif self.recording: + replay_dir = tempfile.mkdtemp(prefix="replay-") + else: + raise IOError("No such file: %s" % repfile) + + self.cache2dir = os.path.join(replay_dir, "httpcache") + self.replay_dir = replay_dir + self.repfile = repfile + self.usedir = usedir + + self.options_path = os.path.join(replay_dir, "options.db") + self.scraped_path = os.path.join(replay_dir, "items_scraped.db") + self.passed_path = os.path.join(replay_dir, "items_passed.db") + self.responses_path = os.path.join(replay_dir, "responses.db") + + if self.updating: + self.options.update(shelve.open(self.options_path)) + #self.responses_old.update(shelve.open(self.responses_path)) + if self.playing: + self.options.update(shelve.open(self.options_path)) + self.responses_old.update(shelve.open(self.responses_path)) + self.scraped_old.update(shelve.open(self.scraped_path)) + self.passed_old.update(shelve.open(self.passed_path)) + + def _persistdb(self, dict_, filename): + d = shelve.open(filename) + d.clear() + d.update(dict_) + d.close() + + def _closedb(self): + if self.recording or self.updating: + d = shelve.open(self.options_path) + for k, v in self.options.iteritems(): + d[k] = v + d.close() + self._persistdb(self.options, self.options_path) + self._persistdb(self.scraped_old, self.scraped_path) + self._persistdb(self.passed_old, self.passed_path) + self._persistdb(self.responses_old, self.responses_path) + + if not self.usedir: + if self.recording or self.updating: + tar = tarfile.open(self.repfile, "w:gz") + for name in os.listdir(self.replay_dir): + tar.add(os.path.join(self.replay_dir, name), name) + tar.close() + + def cleanup(self): + if not self.usedir and os.path.exists(self.replay_dir): + shutil.rmtree(self.replay_dir) diff --git a/scrapy/trunk/scrapy/spider/__init__.py b/scrapy/trunk/scrapy/spider/__init__.py new file mode 100644 index 000000000..2a88816de --- /dev/null +++ b/scrapy/trunk/scrapy/spider/__init__.py @@ -0,0 +1,4 @@ +from scrapy.spider.models import BaseSpider +from scrapy.spider.manager import SpiderManager + +spiders = SpiderManager() diff --git a/scrapy/trunk/scrapy/spider/manager.py b/scrapy/trunk/scrapy/spider/manager.py new file mode 100644 index 000000000..b6c14f166 --- /dev/null +++ b/scrapy/trunk/scrapy/spider/manager.py @@ -0,0 +1,137 @@ +""" +SpiderManager is the class which locates and manages all website-specific +spiders +""" +import sys +import os +import urlparse + +from twisted.plugin import getCache + +from scrapy.spider.models import ISpider +from scrapy.core import log +from scrapy.conf import settings +from scrapy.utils.url import url_is_from_spider + +class SpiderManager(object): + """Spider locator and manager""" + + def __init__(self): + self.loaded = False + self.default_domain = None + self.spider_modules = settings.getlist('SPIDER_MODULES') + + @property + def all(self): + self._load_on_demand() + return self._alldict.values() + + @property + def enabled(self): + self._load_on_demand() + return self._enableddict.values() + + def fromdomain(self, domain_name, include_disabled=True): + return self.asdict(include_disabled=include_disabled).get(domain_name) + + def fromurl(self, url, include_disabled=True): + self._load_on_demand() + domain = urlparse.urlparse(url).hostname + domain = str(domain).replace('www.', '') + if domain: + if domain in self._alldict: # try first locating by domain + return self._alldict[domain] + else: # else search spider by spider + plist = self.all if include_disabled else self.enabled + for p in plist: + if url_is_from_spider(url, p): + return p + return self._alldict.get(self.default_domain) + + def asdict(self, include_disabled=True): + self._load_on_demand() + return self._alldict if include_disabled else self._enableddict + + def _load_on_demand(self): + if not self.loaded: + self.load() + + def _enabled_spiders(self): + if settings['ENABLED_SPIDERS']: + return set(settings['ENABLED_SPIDERS']) + elif settings['ENABLED_SPIDERS_FILE']: + if os.path.exists(settings['ENABLED_SPIDERS_FILE']): + lines = open(settings['ENABLED_SPIDERS_FILE']).readlines() + return set([l.strip() for l in lines if not l.strip().startswith('#')]) + else: + return set() + else: + return set() + + def load(self): + self._invaliddict = {} + self._alldict = {} + self._enableddict = {} + enabled_spiders = self._enabled_spiders() + + modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] + for module in modules: + for spider in self._getspiders(ISpider, module): + try: + ISpider.validateInvariants(spider) + self._alldict[spider.domain_name] = spider + if spider.domain_name in enabled_spiders: + self._enableddict[spider.domain_name] = spider + except Exception, e: + self._invaliddict[spider.domain_name] = spider + # we can't use the log module here because it may not be available yet + print "WARNING: Could not load spider %s: %s" % (spider, e) + self.loaded = True + + def reload(self, skip_domains=None): + """ + Reload all enabled spiders. + + This discovers any spiders added under the spiders module/packages, + removes any spiders removed, updates all enabled spiders code and + updates list of enabled spiders from ENABLED_SPIDERS_FILE or + ENABLED_SPIDERS setting. + + Disabled spiders are intentionally excluded to avoid + syntax/initialization errors. Currently running spiders are also + excluded to avoid inconsistent behaviours. + + If skip_domains is passed those spiders won't be reloaded. + + """ + skip_domains = set(skip_domains or []) + modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] + for m in modules: + reload(m) + self.load() # first call to update list of enabled spiders + reloaded = 0 + pdict = self.asdict(include_disabled=False) + for domain, spider in pdict.iteritems(): + if not domain in skip_domains: + reload(sys.modules[spider.__module__]) + reloaded += 1 + self.load() # second call to update spider instances + log.msg("Reloaded %d/%d scrapy spiders" % (reloaded, len(pdict)), level=log.DEBUG) + + def _getspiders(self, interface, package): + """ + This is an override of twisted.plugin.getPlugin, because we're + interested in catching exceptions thrown when loading spiders such as + KeyboardInterrupt + """ + + try: + allDropins = getCache(package) + for dropin in allDropins.itervalues(): + for plugin in dropin.plugins: + adapted = interface(plugin, None) + if adapted is not None: + yield adapted + except KeyboardInterrupt: + sys.stderr.write("Interrupted while loading Scrapy spiders\n") + sys.exit(2) diff --git a/scrapy/trunk/scrapy/spider/middleware.py b/scrapy/trunk/scrapy/spider/middleware.py new file mode 100644 index 000000000..379e74c26 --- /dev/null +++ b/scrapy/trunk/scrapy/spider/middleware.py @@ -0,0 +1,122 @@ +""" +Spider middleware manager +""" + +import types + +from twisted.python.failure import Failure + +from scrapy.core import log +from scrapy.core.exceptions import NotConfigured +from scrapy.utils.misc import load_class, mustbe_deferred, deferred_degenerate +from scrapy.conf import settings + +class SpiderMiddlewareManager(object): + def __init__(self, callback=None, errback=None): + self.loaded = False + self.spider_middleware = [] + self.result_middleware = [] + self.exception_middleware = [] + self.filter_middleware = [] + self.domaininfo = {} + + self.callback = callback or self._callback + self.errback = errback or self._errback + self.load() + + def _add_middleware(self, mw): + if hasattr(mw, 'process_scrape'): + self.spider_middleware.append(mw.process_scrape) + if hasattr(mw, 'process_result'): + self.result_middleware.insert(0, mw.process_result) + if hasattr(mw, 'process_exception'): + self.exception_middleware.insert(0, mw.process_exception) + if hasattr(mw, 'filter'): + self.filter_middleware.insert(0, mw.filter) + + def load(self): + """Load middleware defined in settings module""" + mws = [] + for mwpath in settings.getlist('SPIDER_MIDDLEWARES') or (): + cls = load_class(mwpath) + if cls: + try: + mw = cls() + self._add_middleware(mw) + mws.append(mw) + except NotConfigured: + pass + log.msg("Enabled spider middlewares: %s" % ", ".join([type(m).__name__ for m in mws])) + self.loaded = True + + def scrape(self, request, response, spider): + fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__) + + def process_scrape(response): + for method in self.spider_middleware: + result = method(response=response, spider=spider) + assert result is None or isinstance(result, (list, tuple)), \ + 'Middleware %s must returns None, list or tuple, got %s ' % \ + (fname(method), type(result)) + if result is not None: + return result + return self.call(request=request, response=response, spider=spider) + + + def process_result(result): + for method in self.result_middleware: + result = method(response=response, result=result, spider=spider) + assert isinstance(result, (list, tuple)), \ + 'Middleware %s must returns list or tuple, got %s ' % \ + (fname(method), type(result)) + return result + + def process_exception(_failure): + exception = _failure.value + for method in self.exception_middleware: + result = method(response=response, exception=exception, spider=spider) + assert result is None or isinstance(result, (list, tuple)), \ + 'Middleware %s must returns None, list or tuple, got %s ' % \ + (fname(method), type(result)) + if result is not None: + return result + return _failure + + def _degenerate(gen): + if isinstance(gen, types.GeneratorType): + return deferred_degenerate(gen) + return gen + + dfd = mustbe_deferred(process_scrape, response) + dfd.addCallback(_degenerate) + dfd.addErrback(process_exception) + dfd.addCallback(process_result) + return dfd + + def call(self, request, response, spider): + if isinstance(response, Exception): + return self.errback(request, Failure(response), spider) + return self.callback(request, response, spider) + + def _callback(self, request, response, spider): + request.deferred.callback(response) + return request.deferred + + def _errback(self, request, failure, spider): + request.deferred.errback(failure) + return request.deferred + + +class DummyMiddleware(object): + def process_scrape(self, response, spider): + pass + + def process_result(self, response, result, spider): + return result + + def process_exception(self, response, exception, spider): + pass + + def filter(self, item): + return True + diff --git a/scrapy/trunk/scrapy/spider/models.py b/scrapy/trunk/scrapy/spider/models.py new file mode 100644 index 000000000..48125133a --- /dev/null +++ b/scrapy/trunk/scrapy/spider/models.py @@ -0,0 +1,95 @@ +""" +Base class for scrapy spiders +""" +from zope.interface import Interface, Attribute, invariant, implements +from twisted.plugin import IPlugin + +from scrapy.core.exceptions import UsageError + +def _valid_start_urls(obj): + """Check the start urls specified are valid""" + # TODO: we should add proper validation here + + ## This commented if is a test i was doing, it also takes in account the spider's base class (if any) + ## it works, but the attribute is checked somewhere else (zope?) and causes to fail + ##if not (obj.start_urls or any([hasattr(base, 'start_urls') for base in list(obj.__class__.__bases__)])): + + if not obj.start_urls: + raise UsageError("A start url is required") + +def _valid_domain_name(obj): + """Check the domain name specified is valid""" + if not obj.domain_name: + raise UsageError("A site domain name is required") + +def _valid_download_delay(obj): + """Check the download delay is valid, if specified""" + delay = getattr(obj, 'download_delay', 0) + if not type(delay) in (int, long, float): + raise UsageError("download_delay must be numeric") + if float(delay) < 0.0: + raise UsageError("download_delay must be positive") + +class ISpider(Interface, IPlugin) : + """Interface to be implemented by site-specific web spiders""" + + start_urls = Attribute( + """A sequence of URLs to retrieve to initiate the spider for this + site. A single URL may also be provided here.""") + + domain_name = Attribute( + """The domain name of the site to be scraped.""") + + download_delay = Attribute( + """Optional delay in seconds to wait between web page downloads. + Note that this delay does not apply to image downloads. + A delay of less than a second can be specified.""") + + user_agent = Attribute( + """Optional User-Agent to use for this domain""") + + invariant(_valid_start_urls) + invariant(_valid_domain_name) + invariant(_valid_download_delay) + + def parse(self, pagedata) : + """This is first called with the data corresponding to start_url. It + must return a (possibly empty) sequence where each element is either: + * A Request object for further processing. + * An object that extends ScrapedItem (defined in scrapeditem module) + * or None (this will be ignored) + + When a Request object is returned, the Request is scheduled, then + downloaded and finally its results is handled to the Request callback. + That callback must behave the same way as this function. + + When a ScrapedItem is returned, it is passed to the transformation pipeline + and finally the destination systems are updated. + + The simplest way to use this is to have a method in your class for each + page type. So each function knows the layout and how to extract data + for a single page (or set of similar pages). A typical class might work + like: + * parse() parses the landing page and returns Requests + for a category() function to parse category pages. + * category() parses the category pages and returns links and callbacks + for a item() function to parse item pages. + * item() parses the item details page and returns objects that + extend ScrapedItem + """ + pass + + def init_domain(self): + """This is first called to initialize domain specific quirks, like + session cookies or login stuff + """ + pass + + +class BaseSpider(object): + """Base class for scrapy spiders. All spiders must inherit from this + class.""" + + implements(ISpider) + domain_name = None + extra_domain_names = [] diff --git a/scrapy/trunk/scrapy/stats/__init__.py b/scrapy/trunk/scrapy/stats/__init__.py new file mode 100644 index 000000000..2b2b2fb61 --- /dev/null +++ b/scrapy/trunk/scrapy/stats/__init__.py @@ -0,0 +1,3 @@ +from scrapy.stats.statscollector import StatsCollector + +stats = StatsCollector() diff --git a/scrapy/trunk/scrapy/stats/corestats.py b/scrapy/trunk/scrapy/stats/corestats.py new file mode 100644 index 000000000..20435a04c --- /dev/null +++ b/scrapy/trunk/scrapy/stats/corestats.py @@ -0,0 +1,69 @@ +""" +Scrapy extension for collecting scraping stats +""" +import os +import getpass +import socket +import datetime + +from pydispatch import dispatcher + +from scrapy.core import signals +from scrapy.stats import stats +from scrapy.conf import settings + +class CoreStats(object): + """Scrapy core stats collector""" + + def __init__(self): + stats.setpath('_envinfo/user', getpass.getuser()) + stats.setpath('_envinfo/host', socket.gethostname()) + stats.setpath('_envinfo/logfile', settings['LOGFILE']) + stats.setpath('_envinfo/pid', os.getpid()) + + dispatcher.connect(self.stats_domain_open, signal=stats.domain_open) + dispatcher.connect(self.stats_domain_closing, signal=stats.domain_closing) + dispatcher.connect(self.item_scraped, signal=signals.item_scraped) + dispatcher.connect(self.item_passed, signal=signals.item_passed) + dispatcher.connect(self.item_dropped, signal=signals.item_dropped) + dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) + dispatcher.connect(self.request_uploaded, signal=signals.request_uploaded) + + def stats_domain_open(self, domain, spider): + stats.setpath('%s/start_time' % domain, datetime.datetime.now()) + stats.setpath('%s/envinfo' % domain, stats.getpath('_envinfo')) + stats.incpath('_global/domain_count/opened') + + def stats_domain_closing(self, domain, spider, status): + stats.setpath('%s/finish_time' % domain, datetime.datetime.now()) + stats.setpath('%s/finish_status' % domain, 'OK' if status == 'finished' else status) + stats.incpath('_global/domain_count/%s' % status) + + def item_scraped(self, item, spider): + stats.incpath('%s/item_scraped_count' % spider.domain_name) + stats.incpath('_global/item_scraped_count') + + def item_passed(self, item, spider, pipe_output): + stats.incpath('%s/item_passed_count' % spider.domain_name) + stats.incpath('_global/item_passed_count') + + def item_dropped(self, item, spider, exception): + reason = exception.__class__.__name__ + stats.incpath('%s/item_dropped_count' % spider.domain_name) + stats.incpath('%s/item_dropped_reasons_count/%s' % (spider.domain_name, reason)) + stats.incpath('_global/item_dropped_count') + + def response_downloaded(self, response, spider): + stats.incpath('%s/response_count' % spider.domain_name) + stats.incpath('%s/response_status_count/%s' % (spider.domain_name, response.status)) + stats.incpath('_global/response_downloaded_count') + + reslen = len(response) + stats.incpath('%s/transfer/downloaded_bytes' % spider.domain_name, reslen) + stats.incpath('_global/transfer/downloaded_bytes', reslen) + + def request_uploaded(self, request, spider): + reqlen = len(request) + stats.incpath('%s/transfer/uploaded_bytes' % spider.domain_name, reqlen) + stats.incpath('_global/transfer/uploaded_bytes', reqlen) + diff --git a/scrapy/trunk/scrapy/stats/statscollector.py b/scrapy/trunk/scrapy/stats/statscollector.py new file mode 100644 index 000000000..6ea3b7685 --- /dev/null +++ b/scrapy/trunk/scrapy/stats/statscollector.py @@ -0,0 +1,76 @@ +""" +Scrapy extension for collecting scraping stats +""" +import pprint + +from pydispatch import dispatcher + +from scrapy.core import signals, log +from scrapy.utils.misc import stats_getpath +from scrapy.conf import settings + +class StatsCollector(dict): + # signal sent after a domain is opened for stats collection and its resorces have been allocated. args: domain + domain_open = object() + # signal sent before a domain is closed, and before the stats are persisted. it can be catched to add additional stats. args: domain + domain_closing = object() + # signal sent after a domain is closed and its resources have been freed. args: domain + domain_closed = object() + + def __init__(self, enabled=None): + self.db = None + self.debug = settings.getbool('STATS_DEBUG') + self.enabled = enabled if enabled is not None else settings.getbool('STATS_ENABLED') + self.cleanup = settings.getbool('STATS_CLEANUP') + + if self.enabled: + if settings['SCRAPING_DB']: + from scrapy.store.db import DomainDataHistory + self.db = DomainDataHistory(settings['SCRAPING_DB'], table_name='domain_data_history') + + dispatcher.connect(self._domain_open, signal=signals.domain_open) + dispatcher.connect(self._domain_closed, signal=signals.domain_closed) + else: + self.setpath = lambda *args, **kwargs: None + self.getpath = lambda path, default=None: default + self.incpath = lambda *args, **kwargs: None + self.delpath = lambda *args, **kwargs: None + + def setpath(self, path, value): + d = self + keys = path.split('/') + for key in keys[:-1]: + if key not in d: + d[key] = {} + d = d[key] + d[keys[-1]] = value + + def getpath(self, path, default=None): + return stats_getpath(self, path, default) + + def delpath(self, path): + d = self + keys = path.split('/') + for key in keys[:-1]: + if key in d: + d = d[key] + else: + return + del d[keys[-1]] + + def incpath(self, path, value_diff=1): + curvalue = self.getpath(path) or 0 + self.setpath(path, curvalue + value_diff) + + def _domain_open(self, domain, spider): + dispatcher.send(signal=self.domain_open, sender=self.__class__, domain=domain, spider=spider) + + def _domain_closed(self, domain, spider, status): + dispatcher.send(signal=self.domain_closing, sender=self.__class__, domain=domain, spider=spider, status=status) + if self.debug: + log.msg(pprint.pformat(self[domain]), domain=domain, level=log.DEBUG) + if self.db: + self.db.put(domain, self[domain]) + if self.cleanup: + del self[domain] + dispatcher.send(signal=self.domain_closed, sender=self.__class__, domain=domain, spider=spider, status=status) diff --git a/scrapy/trunk/scrapy/store/__init__.py b/scrapy/trunk/scrapy/store/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/store/db.py b/scrapy/trunk/scrapy/store/db.py new file mode 100644 index 000000000..4f3fbc26f --- /dev/null +++ b/scrapy/trunk/scrapy/store/db.py @@ -0,0 +1,130 @@ +""" +Store implementations using a MySQL DB +""" +import pickle +from datetime import datetime + +from scrapy.utils.misc import stats_getpath +from scrapy.utils.db import mysql_connect + +class DomainDataHistory(object): + """ + This is a store for domain data, with history. + """ + + def __init__(self, db_uri, table_name): + self.db_uri = db_uri + self._mysql_conn = None + self._table = table_name + + def get_mysql_conn(self): + if self._mysql_conn is None: + self._mysql_conn = mysql_connect(self.db_uri, use_unicode=False) + return self._mysql_conn + mysql_conn = property(get_mysql_conn) + + def get(self, domain, count=1, offset=0, stored_after=None, stored_before=None, path=None): + """ + Get the last records stored for the given domain. + If count is given that number of records is retunred, otherwise all records. + If stored_{after,before} is given the results are restricted to that time interval. + The result is a list of tuples: (timestamp, data) + If path is given it only returns stats of that given path (only for dict objects like stats) + """ + sqlsuf = "" + if count is not None: + sqlsuf += "LIMIT %d " % count + if offset: + sqlsuf += "OFFSET %d " % offset + filter = "" + if stored_after: + filter += "AND stored>='%s'" % stored_after.strftime("%Y-%m-%d %H:%M:%S") + if stored_before: + filter += "AND stored<='%s'" % stored_before.strftime("%Y-%m-%d %H:%M:%S") + + c = self.mysql_conn.cursor() + select = "SELECT stored,data FROM %s WHERE domain=%%s %s ORDER BY stored DESC %s" % (self._table, filter, sqlsuf) + c.execute(select, domain) + for stored, datas in c: + obj = pickle.loads(datas) + if path and isinstance(obj, dict): + yield stored, stats_getpath(obj, path) + else: + yield stored, obj + + def getlast(self, domain, offset=0, path=None): + """ + Get the Nth last data stored for the given domain + If offset=0 get the last data stored + If offset=1 get the previous data stored + ...and so on + """ + datal = list(self.get(domain, count=1, offset=offset, path=path)) + if datal: + return datal[0] + + def getall(self, domain, path=None): + """ + Get all data stored for the given domain as a list of tuples: + (stored, timestamp) + """ + return self.get(domain, count=None, path=path) + + def getlast_alldomains(self, count=None, offset=None, order='domain', olist='ASC', path=None): + """ + Get all last data for all domains as a list of tuples, unless + data is sliced by count and offset values. + Ordered by domain is the default. + TODO: This could be merged with: get(domain=None, ..., +order) + + Return a list of tuples (domain, timestamp, data) + """ + sqlsuf = "" + if count is not None: + sqlsuf += "LIMIT %s " % count + if offset: + sqlsuf += "OFFSET %s " % offset + + c = self.mysql_conn.cursor() + select = "SELECT domain, MAX(stored) as stored, data \ + FROM ( \ + SELECT * FROM %s ORDER BY stored DESC \ + ) AS inner_ordered \ + GROUP BY domain \ + ORDER BY %s %s %s" % (self._table, order, olist, sqlsuf) + c.execute(select) + for domain, stored, datas in c: + obj = pickle.loads(datas) + if path and isinstance(obj, dict): + yield domain, stored, stats_getpath(obj, path) + else: + yield domain, stored, obj + + def domain_count(self): + """ + Return the number of domains stored in the database + """ + c = self.mysql_conn.cursor() + c.execute("SELECT COUNT(DISTINCT(domain)) FROM %s" % self._table) + return c.fetchone()[0] + + def put(self, domain, data, timestamp=None): + """ + Store data in history using the given timestamp (or now if omitted). + data can be any pickable object + """ + stored = timestamp if timestamp else datetime.now() + datas = pickle.dumps(data) + + c = self.mysql_conn.cursor() + c.execute("INSERT INTO %s (domain,stored,data) VALUES (%%s,%%s,%%s)" % self._table, (domain, stored, datas)) + self.mysql_conn.commit() + + def remove(self, domain): + """ + Remove all data stored for the given domain + """ + c = self.mysql_conn.cursor() + c.execute("DELETE FROM %s WHERE domain=%%s" % self._table, domain) + self.mysql_conn.commit() + diff --git a/scrapy/trunk/scrapy/tests/__init__.py b/scrapy/trunk/scrapy/tests/__init__.py new file mode 100644 index 000000000..b72b1493f --- /dev/null +++ b/scrapy/trunk/scrapy/tests/__init__.py @@ -0,0 +1 @@ +""" """ \ No newline at end of file diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml new file mode 100644 index 000000000..7849e5631 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample1.xml @@ -0,0 +1,100 @@ + + + + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    Picture can be personalised!

    An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!)

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12")

    ]]>     + + +     + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    Picture can be personalised!

    An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!)

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12")

    ]]>     + + +     + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    Picture can be personalised!

    An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!)

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12")

    ]]>     + + +     + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    Picture can be personalised!

    An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!)

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12")

    ]]>     + + +     + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    Picture can be personalised!

    An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!)

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12")

    ]]>     + + +     + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    An original and delightful christening gift , perfect nursery decoration.

    FREE hand finished, "Quiet night" greeting card included!

    Picture can be personalised!

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12").

    ]]>     + + +     + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    An original and delightful christening gift , perfect nursery decoration.

    FREE hand finished, "Sunshine lullaby" greeting card included!

    Picture can be personalised!

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12").

    ]]>     + + +     + + + + + 28.00 + + + + Beautiful, open edition giclee print from Agatha's original watercolours, reprinted onto watercolour paper, reproducing all the qualities of the original painting.

    The print is conservation mounted and individually signed by the artist.

    Picture can be personalised!

    An original and delightful gift, perfect as a bedroom or nursery decoration. (Especially pleasing for any cat lover!)

    Picture size: 17cm x 17cm Framed picture size: 30cm x 30cm (12" x 12")

    ]]>     + + +     +     diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml new file mode 100644 index 000000000..0261ae903 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample2.xml @@ -0,0 +1,100 @@ + + + +Example.com +Example.com +http://www.example.com/ +Abadeh 7500.924 - 67cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1602 +http://www.example.com/uploads/images_products/1602.jpg + +69 + +Abadeh 7500.924 - 83cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1602 +http://www.example.com/uploads/images_products/1602.jpg + +89 + +Abadeh 7500.929 - 83cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1603 +http://www.example.com/uploads/images_products/1603.jpg + +89 + +Abadeh 7500.929 - 67cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1603 +http://www.example.com/uploads/images_products/1603.jpg + +69 + +Abadeh 7500.938 - 67cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1604 +http://www.example.com/uploads/images_products/1604.jpg + +69 + +Abadeh 7500.938 - 83cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1604 +http://www.example.com/uploads/images_products/1604.jpg + +89 + +Abadeh 7500.944 - 83cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1605 +http://www.example.com/uploads/images_products/1605.jpg + +89 + +Abadeh 7500.944 - 67cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1605 +http://www.example.com/uploads/images_products/1605.jpg + +69 + +Abadeh 7501.938 - 67cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=939 +http://www.example.com/uploads/images_products/939.jpg + +69 + +Abadeh 7501.938 - 83cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=939 +http://www.example.com/uploads/images_products/939.jpg + +89 + +Abadeh 7501.958 - 83cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1606 +http://www.example.com/uploads/images_products/1606.jpg + +89 + +Abadeh 7501.958 - 67cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=1606 +http://www.example.com/uploads/images_products/1606.jpg + +69 + +Abadeh 7502.938 - 67cm Wide +<strong>Price shown per linear metre<br><br>Minimum purchase 3 linear metres</strong><br><br>Quantity discounts available on 30m rolls.<br><br> +http://www.example.com/product.asp?P_ID=936 +http://www.example.com/uploads/images_products/936.jpg + +69 + + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample3.xml b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample3.xml new file mode 100644 index 000000000..ddbe8d4db Binary files /dev/null and b/scrapy/trunk/scrapy/tests/sample_data/feeds/feed-sample3.xml differ diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extraction/extract_urls.html b/scrapy/trunk/scrapy/tests/sample_data/link_extraction/extract_urls.html new file mode 100644 index 000000000..ad5c7f128 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/link_extraction/extract_urls.html @@ -0,0 +1,49 @@ + + + + + +Eco-Friendly Pillow + + + + + + + + + + + + +
    + +
    +
    +
    + Shopping Cart
    + 0 items +
    line +
    +
    Why Go Eco Friendly?
    +
    +
    +

    Our Eco Friendly Products

    +
    Eco Duvet

    Eco Duvet

    from £30.80

    Eco Mattress Protector

    Eco Mattress Protector

    from £25.40

    Eco Pillow

    Eco Pillow

    from £22.00

    +
    +
    + +
    + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/test_site/index.html b/scrapy/trunk/scrapy/tests/sample_data/test_site/index.html new file mode 100644 index 000000000..d268c846a --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/test_site/index.html @@ -0,0 +1,18 @@ + + + +Scrapy test site + + + + +

    Scrapy test site

    + + + + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html b/scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html new file mode 100644 index 000000000..ceeb6dc87 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/test_site/item1.html @@ -0,0 +1,17 @@ + + + +Item 1 - Scrapy test site + + + + +

    Item 1 name

    + +
      +
    • Price: $100
      • +
    • Stock: 12
      • +
    + + + diff --git a/scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html b/scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html new file mode 100644 index 000000000..a64c92810 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/test_site/item2.html @@ -0,0 +1,17 @@ + + + +Item 2 - Scrapy test site + + + + +

    Item 2 name

    + +
      +
    • Price: $200
      • +
    • Stock: 5
      • +
    + + + diff --git a/scrapy/trunk/scrapy/tests/test_c14nurls.py b/scrapy/trunk/scrapy/tests/test_c14nurls.py new file mode 100644 index 000000000..e0a8e8166 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_c14nurls.py @@ -0,0 +1,28 @@ +import unittest +from scrapy.utils.c14n import canonicalize + +class C14nTest(unittest.TestCase): + """ c14n comparison functions """ + + def test_canonicalize(self): + """Test URL canonicalization function""" + urls = [ + ('http://www.maddiebrown.co.uk//cgi-bin//html_parser.cgi?web_page_id=12&template_file=catalogue_list_items_template&category-id=1000070&category-type=1&page=3&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2&page=2', + 'http://www.maddiebrown.co.uk//cgi-bin//html_parser.cgi?web_page_id=12&template_file=catalogue_list_items_template&category-id=1000070&category-type=1&page=3&page=2'), + + + ('http://www.mfi.co.uk/mfi/productinfo.asp?CT=/1010/YourHomeOffice&CT=/1025/YourHomeOffice/Packages', + 'http://www.mfi.co.uk/mfi/productinfo.asp?CT=/1010/YourHomeOffice&CT=/1025/YourHomeOffice/Packages'), + + ('http://www.test-nice-url.com/index.html', + 'http://www.test-nice-url.com/index.html'), + + ('http://www.homebase.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=20001&langId=-1&catalogId=10701&productId=729482&Trail=C$cip=50704&categoryId=50704', + 'http://www.homebase.co.uk/webapp/wcs/stores/servlet/ProductDisplay?storeId=20001&langId=-1&catalogId=10701&productId=729482&Trail=C$cip=50704&categoryId=50704'), + ] + for origurl, c14nurl in urls: + self.assertEqual(canonicalize(origurl), c14nurl) + +if __name__ == "__main__": + unittest.main() + diff --git a/scrapy/trunk/scrapy/tests/test_defaultencoding.py b/scrapy/trunk/scrapy/tests/test_defaultencoding.py new file mode 100644 index 000000000..36c5b782d --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_defaultencoding.py @@ -0,0 +1,9 @@ +import sys +from unittest import TestCase, main + +class DefaultEncodingTest(TestCase): + def test_defaultencoding(self): + self.assertEqual(sys.getdefaultencoding(), 'utf-8') + +if __name__ == "__main__": + main() diff --git a/scrapy/trunk/scrapy/tests/test_dependencies.py b/scrapy/trunk/scrapy/tests/test_dependencies.py new file mode 100644 index 000000000..5109edee8 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_dependencies.py @@ -0,0 +1,16 @@ +from unittest import TestCase, main + +class ScrapyUtilsTest(TestCase): + def test_openssl(self): + try: + module = __import__('OpenSSL', {}, {}, ['']) + except ImportError, ex: + return # no openssl installed + + required_version = '0.6' + if hasattr(module, '__version__'): + for cur, req in zip(module.__version__.split('.'), required_version.split('.')): + self.assertFalse(cur < req, "module %s >= %s required" % ('OpenSSL', required_version)) + +if __name__ == "__main__": + main() diff --git a/scrapy/trunk/scrapy/tests/test_engine.py b/scrapy/trunk/scrapy/tests/test_engine.py new file mode 100644 index 000000000..f64a8062a --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_engine.py @@ -0,0 +1,197 @@ +""" +Scrapy engine tests +""" + +import sys +import os +import urlparse +import unittest + +from twisted.internet import reactor +from twisted.web import server, resource, static, util + +#class TestResource(resource.Resource): +# isLeaf = True +# +# def render_GET(self, request): +# return "hello world!" + +def start_test_site(): + root_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sample_data", "test_site") + r = static.File(root_dir) +# r.putChild("test", TestResource()) + r.putChild("redirect", util.Redirect("/redirected")) + r.putChild("redirected", static.Data("Redirected here", "text/plain")) + + port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") + return port + + +class CrawlingSession(object): + + def __init__(self): + self.domain = 'scrapytest.org' + self.spider = None + self.respplug = [] + self.reqplug = [] + self.itemresp = [] + self.signals_catched = {} + self.wasrun = False + + def run(self): + self.port = start_test_site() + self.portno = self.port.getHost().port + + from scrapy.spider import spiders + spiders.spider_modules = ['scrapy.tests.test_spiders'] + spiders.reload() + + self.spider = spiders.fromdomain(self.domain) + if self.spider: + self.spider.start_urls = [ + self.geturl("/"), + self.geturl("/redirect"), + ] + + from scrapy.core import signals + from scrapy.core.manager import scrapymanager + from scrapy.core.engine import scrapyengine + from pydispatch import dispatcher + + dispatcher.connect(self.record_signal, signals.engine_started) + dispatcher.connect(self.record_signal, signals.engine_stopped) + dispatcher.connect(self.record_signal, signals.domain_opened) + dispatcher.connect(self.record_signal, signals.domain_idle) + dispatcher.connect(self.record_signal, signals.domain_closed) + dispatcher.connect(self.item_scraped, signals.item_scraped) + dispatcher.connect(self.request_received, signals.request_received) + dispatcher.connect(self.response_downloaded, signals.response_downloaded) + + scrapymanager.runonce(self.domain) + self.port.stopListening() + self.wasrun = True + + def geturl(self, path): + return "http://localhost:%s%s" % (self.portno, path) + + def getpath(self, url): + u = urlparse.urlparse(url) + return u.path + + def item_scraped(self, item, spider, response): + self.itemresp.append((item, response)) + + def request_received(self, request, spider): + self.reqplug.append((request, spider)) + + def response_downloaded(self, response, spider): + self.respplug.append((response, spider)) + + def record_signal(self, *args, **kwargs): + """Record a signal and its parameters""" + signalargs = kwargs.copy() + sig = signalargs.pop('signal') + signalargs.pop('sender', None) + self.signals_catched[sig] = signalargs + +session = CrawlingSession() + + +class EngineTest(unittest.TestCase): + + def setUp(self): + if not session.wasrun: + session.run() + + # disable extensions that cause problems with tests (probably + # because they leave the reactor in an unclean state) + from scrapy.conf import settings + settings.overrides['CLUSTER_MANAGER_ENABLED'] = 0 + + def test_spider_locator(self): + """ + Check the spider is loaded and located properly via the SpiderLocator + """ + assert session.spider is not None + self.assertEqual(session.spider.domain_name, session.domain) + + def test_visited_urls(self): + """ + Make sure certain URls were actually visited + """ + # expected urls that should be visited + must_be_visited = ["/", "/redirect", "/redirected", + "/item1.html", "/item2.html", "/item999.html"] + + urls_visited = set([rp[0].url for rp in session.respplug]) + urls_expected = set([session.geturl(p) for p in must_be_visited]) + assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) + + def test_requests_received(self): + """ + Check requests received + """ + # 3 requests should be received from the spider. start_urls and redirects don't count + self.assertEqual(3, len(session.reqplug)) + + paths_expected = ['/item999.html', '/item2.html', '/item1.html'] + + urls_requested = set([rq[0].url for rq in session.reqplug]) + urls_expected = set([session.geturl(p) for p in paths_expected]) + assert urls_expected <= urls_requested + + def test_responses_downloaded(self): + """ + Check responses downloaded + """ + # response tests + self.assertEqual(6, len(session.respplug)) + + for response, spider in session.respplug: + if session.getpath(response.url) == '/item999.html': + self.assertEqual('404', response.status) + if session.getpath(response.url) == '/redirect': + self.assertEqual('302', response.status) + self.assertEqual(response.domain, spider.domain_name) + + def test_item_data(self): + """ + Check item data + """ + # item tests + self.assertEqual(2, len(session.itemresp)) + for item, response in session.itemresp: + self.assertEqual(item.url, response.url) + if 'item1.html' in item.url: + self.assertEqual('Item 1 name', item.name) + self.assertEqual('100', item.price) + if 'item2.html' in item.url: + self.assertEqual('Item 2 name', item.name) + self.assertEqual('200', item.price) + + def test_signals(self): + """ + Check signals were sent properly + """ + from scrapy.core import signals + + assert signals.engine_started in session.signals_catched + assert signals.engine_stopped in session.signals_catched + assert signals.domain_opened in session.signals_catched + assert signals.domain_idle in session.signals_catched + assert signals.domain_closed in session.signals_catched + + self.assertEqual({'domain': session.domain, 'spider': session.spider}, + session.signals_catched[signals.domain_opened]) + self.assertEqual({'domain': session.domain, 'spider': session.spider}, + session.signals_catched[signals.domain_idle]) + self.assertEqual({'domain': session.domain, 'spider': session.spider, 'status': 'finished'}, + session.signals_catched[signals.domain_closed]) + +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == 'runserver': + port = start_test_site() + print "Test server running at http://localhost:%d/ - hit Ctrl-C to finish." % port.getHost().port + reactor.run() + else: + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py new file mode 100644 index 000000000..511ca9797 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_http_request.py @@ -0,0 +1,230 @@ +import unittest +from scrapy.http import Request, Headers +from scrapy.core.scheduler import GroupFilter + +class RequestTest(unittest.TestCase): + """ c14n comparison functions """ + + def test_groupfilter(self): + k1 = Request(url="http://www.scrapy.org/path?key=val").fingerprint() + k2 = Request(url="http://www.scrapy.org/path?key=val&key=val").fingerprint() + self.assertEqual(k1, k2) + + f = GroupFilter() + f.open("mygroup") + self.assertTrue(f.add("mygroup", k1)) + self.assertFalse(f.add("mygroup", k1)) + self.assertFalse(f.add("mygroup", k2)) + + f.open('anothergroup') + self.assertTrue(f.add("anothergroup", k1)) + self.assertFalse(f.add("anothergroup", k1)) + self.assertFalse(f.add("anothergroup", k2)) + + f.close('mygroup') + f.open('mygroup') + self.assertTrue(f.add("mygroup", k2)) + self.assertFalse(f.add("mygroup", k1)) + + def test_headers(self): + # Different ways of setting headers attribute + url = 'http://www.scrapy.org' + headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'} + r = Request(url=url, headers=headers) + p = Request(url=url, headers=r.headers) + + self.assertEqual(r.headers, p.headers) + self.assertFalse(r.headers is headers) + self.assertFalse(p.headers is r.headers) + + # headers must not be unicode + h = Headers({'key1': u'val1', u'key2': 'val2'}) + h[u'newkey'] = u'newval' + for k,v in h.iteritems(): + self.assert_(isinstance(k, str)) + self.assert_(isinstance(v, str)) + + r = Request(url="http://www.example.org/1", referer=u"http://www.example.org") + self.assert_(isinstance(r.headers['referer'], str)) + + def test_eq(self): + url = 'http://www.scrapy.org' + r1 = Request(url=url) + r2 = Request(url=url) + self.assertNotEqual(r1, r2) + + set_ = set() + set_.add(r1) + set_.add(r2) + self.assertEqual(len(set_), 2) + + def test_fingerprint(self): + url = 'http://www.scrapy.org' + r = Request(url=url) + urlhash = r.fingerprint() + + # fingerprint including all initials headers + r.fingerprint_params['exclude_headers'] = [] + fullhash = r.fingerprint() + del r.fingerprint_params['exclude_headers'] + + # all headers are excluded from fingerprint by default + r.headers['Accept'] = 'application/json' + accepthash = r.fingerprint() + r.headers['User-Agent'] = 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20071004 Iceweasel/2.0.0.8 (Debian-2.0.0.8-1)' + useragenthash = r.fingerprint() + self.assertEqual(urlhash, accepthash) + self.assertEqual(urlhash, useragenthash) + self.assertEqual(accepthash, useragenthash) + + # take User-Agent header in count as specified in `include_headers` + r.fingerprint_params['include_headers'] = ['user-agent'] + useragenthash = r.fingerprint() + self.assertNotEqual(useragenthash, urlhash) + + # include_headers has precedence over exclude_headers + r.fingerprint_params['exclude_headers'] = ['user-agent', 'Accept'] + self.assertEqual(useragenthash, r.fingerprint()) + + del r.fingerprint_params['include_headers'] + self.assertNotEqual(useragenthash, r.fingerprint()) # exclude_headers is excluding 'User-Agent' from hash + self.assertEqual(fullhash, r.fingerprint()) # all headers previously seted was excluded + + # set more headers + r.headers['Accept-Language'] = 'en-us,en;q=0.5' + r.headers['SESSIONID'] = 'an ugly session id header' + self.assertNotEqual(urlhash, r.fingerprint()) + + # force emtpy include_headers (ignore exclude_headers) + r.fingerprint_params['include_headers'] = [] + self.assertEqual(urlhash, r.fingerprint()) + + # Tamper Function + r.fingerprint_params['tamperfunc'] = lambda req: Request(url=req.url) + self.assertEqual(urlhash, r.fingerprint()) + + # Compare request with None vs {} header + r = Request(url=url, headers=None) + o = Request(url=url, headers={}) + self.assertEqual(o.fingerprint(), r.fingerprint()) + self.assertNotEqual(r, o) + + # Different ways of setting headers attribute + headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'} + r = Request(url=url, headers=headers) + p = Request(url=url, headers=r.headers) + o = Request(url=url) + o.headers = headers + self.assertEqual(r.fingerprint(), o.fingerprint()) + self.assertEqual(r.fingerprint(), p.fingerprint()) + self.assertEqual(o.fingerprint(), p.fingerprint()) + self.assertNotEqual(r, o) + self.assertNotEqual(r, p) + self.assertNotEqual(o, p) + + # same url, implicit method + r1 = Request(url=url) + r2 = Request(url=url, method='GET') + self.assertEqual(r1.fingerprint(), r2.fingerprint()) + + # same url, different method + r3 = Request(url=url, method='POST') + self.assertNotEqual(r1.fingerprint(), r3.fingerprint()) + + # implicit POST method + r3.body = '' + r4 = Request(url=url, body='') + self.assertEqual(r3.fingerprint(), r4.fingerprint()) + + # body is not important in GET or DELETE + r1 = Request(url=url, method='get', body='data') + r2 = Request(url=url, method='get') + self.assertEqual(r1.fingerprint(), r2.fingerprint()) + + r1 = Request(url=url, method='delete', body='data') + r2 = Request(url=url, method='delete') + self.assertEqual(r1.fingerprint(), r2.fingerprint()) + + # no body by default + r1 = Request(url=url, method='POST') + r2 = Request(url=url, method='POST', body=None) + self.assertEqual(r1.fingerprint(), r2.fingerprint()) + + # empty body is equal to None body + r3 = Request(url=url, method='POST', body='') + self.assertEqual(r1.fingerprint(), r3.fingerprint()) + + def test_insensitive_request_fingerprints(self): + url = 'http://www.scrapy.org' + fp = {'include_headers':['Accept','Content-Type']} + r1a = Request(url=url.lower()) + r1b = Request(url=url.upper()) + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + + r1a = Request(url=url.lower(), method='get') + r1b = Request(url=url.upper(), method='GET') + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + + r1c = Request(url=url.upper(), method='GET', body='this is not important') + self.assertEqual(r1b.fingerprint(), r1c.fingerprint()) + + r1a = Request(url=url.lower(), method='get', fingerprint_params=fp) + r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp) + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + + r1a = Request(url=url.lower(), method='get', headers={'ACCEPT':'Black'}) + r1b = Request(url=url.upper(), method='GET', headers={'ACCEPT':'Black'}) + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + + r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'}) + r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'ACCEPT':'Black'}) + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + + r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black'}) + r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black'}) + r1c = Request(url=url.upper(), method='gEt', fingerprint_params=fp, headers={'accept':'Black'}) + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + self.assertEqual(r1a.fingerprint(), r1c.fingerprint()) + + r1a = Request(url=url.lower(), method='get', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'}) + r1b = Request(url=url.upper(), method='GET', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'}) + r1c = Request(url=url.upper(), method='Get', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'}) + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + self.assertEqual(r1a.fingerprint(), r1c.fingerprint()) + + r1a = Request(url=url.lower(), method='Post', fingerprint_params=fp, headers={'ACCEPT':'Black', 'content-type':'application/json'}) + r1b = Request(url=url.upper(), method='POST', fingerprint_params=fp, headers={'Accept':'Black', 'Content-Type':'application/json'}) + r1c = Request(url=url.upper(), method='posT', fingerprint_params=fp, headers={'accepT':'Black', 'CONTENT-TYPE':'application/json'}) + self.assertEqual(r1a.fingerprint(), r1b.fingerprint()) + self.assertEqual(r1a.fingerprint(), r1c.fingerprint()) + + def test_url(self): + """Request url tests""" + r = Request(url="http://www.scrapy.org/path") + self.assertEqual(r.url, "http://www.scrapy.org/path") + + # url quoting on attribute assign + r.url = "http://www.scrapy.org/blank%20space" + self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + r.url = "http://www.scrapy.org/blank space" + self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + + # url quoting on creation + r = Request(url="http://www.scrapy.org/blank%20space") + self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + r = Request(url="http://www.scrapy.org/blank space") + self.assertEqual(r.url, "http://www.scrapy.org/blank%20space") + + # url coercion to string + r.url = u"http://www.scrapy.org/test" + self.assert_(isinstance(r.url, str)) + + # url encoding + r = Request(url=u"http://www.scrapy.org/price/\xa3", url_encoding="utf-8") + self.assert_(isinstance(r.url, str)) + self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + +if __name__ == "__main__": + unittest.main() + + diff --git a/scrapy/trunk/scrapy/tests/test_http_response.py b/scrapy/trunk/scrapy/tests/test_http_response.py new file mode 100644 index 000000000..5c3a7692f --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_http_response.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +from unittest import TestCase, main +import libxml2 +from scrapy.http import ResponseBody + +class ResponseTest(TestCase): + def test_responsebodyencoding(self): + string = 'кириллический текст' + unicode_string = unicode(string, 'utf-8') + body_cp1251 = unicode_string.encode('cp1251') + body = ResponseBody(body_cp1251, 'cp1251') + self.assertEqual(body.to_unicode(), unicode_string) + self.assertEqual(isinstance(body.to_unicode(), unicode), True) + self.assertEqual(body.to_string('utf-8'), string) + self.assertEqual(isinstance(body.to_string('utf-8'), str), True) + self.assertEqual(body.to_string(), body_cp1251) + self.assertEqual(isinstance(body.to_string('utf-8'), str), True) + +if __name__ == "__main__": + main() diff --git a/scrapy/trunk/scrapy/tests/test_http_url.py b/scrapy/trunk/scrapy/tests/test_http_url.py new file mode 100644 index 000000000..08266a262 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_http_url.py @@ -0,0 +1,28 @@ +import unittest +from scrapy.http import Url + +class UrlClassTest(unittest.TestCase): + + def test_url_attributes(self): + u = Url("http://scrapy.org/wiki/info") + self.assertEqual("scrapy.org", u.hostname) + self.assertEqual("/wiki/info", u.path) + self.assertEqual(None, u.username) + self.assertEqual(None, u.password) + + u = Url("http://someuser:somepass@example.com/ticket/query?owner=pablo") + self.assertEqual("someuser", u.username) + self.assertEqual("somepass", u.password) + self.assertEqual("example.com", u.hostname) + self.assertEqual("/ticket/query", u.path) + self.assertEqual("owner=pablo", u.query) + + u = Url("http://example.com/somepage.html#fragment-1") + self.assertEqual("fragment-1", u.fragment) + + u = Url("file:///home/pablo/file.txt") + self.assertEqual("/home/pablo/file.txt", u.path) + +if __name__ == "__main__": + unittest.main() + diff --git a/scrapy/trunk/scrapy/tests/test_libxml2.py b/scrapy/trunk/scrapy/tests/test_libxml2.py new file mode 100644 index 000000000..35b0cade4 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_libxml2.py @@ -0,0 +1,43 @@ +from unittest import TestCase, main +import libxml2 +from scrapy.http import ResponseBody, Response + +class Libxml2Test(TestCase): + def setUp(self): + libxml2.debugMemory(1) + + def tearDown(self): + libxml2.cleanupParser() + leaked_bytes = libxml2.debugMemory(0) + assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes + + def test_xpath(self): + #this test will fail in version 2.6.27 but passes on 2.6.29+ + html = "    " + node = libxml2.htmlParseDoc(html, 'utf-8') + result = [str(r) for r in node.xpathEval('//text()')] + self.assertEquals(result, ['1', '2', '3']) + node.freeDoc() + +class ResponseLibxml2DocTest(TestCase): + def setUp(self): + libxml2.debugMemory(1) + + def tearDown(self): + libxml2.cleanupParser() + leaked_bytes = libxml2.debugMemory(0) + assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes + + def test_getlibxml2doc(self): + # test to simulate '\x00' char in body of html page + #this method don't should raise TypeError Exception + from scrapy.core.manager import scrapymanager + scrapymanager.configure() + + self.body_content = 'test problematic \x00 body' + self.problematic_body = ResponseBody(self.body_content, 'utf-8') + response = Response('example.com', 'http://example.com/catalog/product/blabla-123', body=self.problematic_body) + response.getlibxml2doc() + +if __name__ == "__main__": + main() diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py new file mode 100644 index 000000000..e4cf3ee58 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -0,0 +1,35 @@ +import unittest + +from scrapy.http import Response +from scrapy.link import LinkExtractor + +class LinkExtractorTestCase(unittest.TestCase): + def test_basic(self): + html = """Page title<title> + <body><p><a href="item/12.html">Item 12</a></p> + <p><a href="/about.html">About us</a></p> + <img src="/logo.png" alt="Company logo (not a link)" /> + <p><a href="../othercat.html">Other category</a></p> + <p><a href="/" /></p> + </body></html>""" + response = Response("example.org", "http://example.org/somepage/index.html", body=html) + + lx = LinkExtractor() # default: tag=a, attr=href + self.assertEqual(lx.extract_urls(response), + {'http://example.org/somepage/item/12.html': 'Item 12', + 'http://example.org/about.html': 'About us', + 'http://example.org/othercat.html': 'Other category', + 'http://example.org/': ''}) + + def test_base_url(self): + html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = Response("example.org", "http://example.org/somepage/index.html", body=html) + + lx = LinkExtractor() # default: tag=a, attr=href + self.assertEqual(lx.extract_urls(response), + {'http://otherdomain.com/base/item/12.html': 'Item 12'}) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_link_extraction.py b/scrapy/trunk/scrapy/tests/test_link_extraction.py new file mode 100644 index 000000000..9a177be55 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_link_extraction.py @@ -0,0 +1,26 @@ +import os +import unittest + +from decobot.utils.link_extraction import extract_urls + +class LinkExtractionTest(unittest.TestCase): + def setUp(self): + self.datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sample_data", "link_extraction") + + def test_extract_urls(self): + text = open(os.path.join(self.datadir, "extract_urls.html")).read() + links_expected = {'/products/eco-mattress-protector.html': 'Eco Mattress Protector', + '/products.html': 'Our Products', + '/products/eco-duvet.html': 'Eco Duvet', + '/terms-and-conditions.html': 'Terms & Conditions', + '/returns-policy.html': 'Returns Policy', + '/products/eco-pillow.html': 'Eco Pillow', + '/': 'Home', + '/privacy-policy.html': 'Privacy Policy', + '/eco-friendly.html': 'Eco-Friendly', + '/faqs.html': 'FAQs', + '/contact-us.html': 'Contact Us'} + self.assertEqual(links_expected, extract_urls(text)) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_serialization.py b/scrapy/trunk/scrapy/tests/test_serialization.py new file mode 100644 index 000000000..1c7d3f18c --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_serialization.py @@ -0,0 +1,22 @@ +import unittest +from scrapy.utils.serialization import serialize, unserialize, serialize_funcs + +class SerializationTest(unittest.TestCase): + def test_string_serialization(self): + """Test simple string serialization""" + + s = "string to serialize" + for format in serialize_funcs.iterkeys(): + self.assertEqual(s, unserialize(serialize(s))) + + def test_dict_serialization(self): + """Test dict serialization""" + + # using only string keys/values since it's the only data type that + # works with all serializers + d = {'one': 'item one', 'two': 'item two'} + for format in serialize_funcs.iterkeys(): + self.assertEqual(d, unserialize(serialize(d))) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_spidermonkey.py b/scrapy/trunk/scrapy/tests/test_spidermonkey.py new file mode 100644 index 000000000..52b753d32 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_spidermonkey.py @@ -0,0 +1,24 @@ +from twisted.trial import unittest + +class SpidermonkeyTest(unittest.TestCase): + + def setUp(self): + try: + from spidermonkey import Runtime + r = Runtime() + self.cx = r.new_context() + except ImportError: + raise unittest.SkipTest("Spidermonkey C library not available") + + def test_spidermonkey(self): + """Spidermonkey basic functionality tests""" + + self.cx.eval_script("var item0={'price':50}") + self.cx.eval_script("var item1={'price':20}") + self.assertEqual(self.cx.eval_script("item0"), {'price': 50}) + self.assertEqual(self.cx.eval_script("item1"), {'price': 20}) + self.cx.eval_script("var itemArray=new Array(item0, item1)") + self.assertEqual(self.cx.eval_script("itemArray"), [{'price': 50}, {'price':20}]) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_spiders/__init__.py b/scrapy/trunk/scrapy/tests/test_spiders/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/tests/test_spiders/testplugin.py b/scrapy/trunk/scrapy/tests/test_spiders/testplugin.py new file mode 100644 index 000000000..0d18485c3 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_spiders/testplugin.py @@ -0,0 +1,36 @@ +""" +This is a spider for the unittest sample site. + +See scrapy/tests/test_engine.py for more info. +""" +import re + +from scrapy.spider import BaseSpider +from scrapy.item import ScrapedItem +from decobot.utils.link_extraction import follow_link_pattern + +class TestSpider(BaseSpider): + domain_name = "scrapytest.org" + extra_domain_names = ["localhost"] + start_urls = ['http://localhost'] + + itemurl_re = re.compile("item\d+.html") + name_re = re.compile("<h1>(.*?)</h1>", re.M) + price_re = re.compile(">Price: \$(.*?)<", re.M) + + def parse(self, response): + return follow_link_pattern(response.body.to_string(), self.parse_item, response.url, self.itemurl_re) + + def parse_item(self, response): + item = ScrapedItem() + m = self.name_re.search(response.body.to_string()) + if m: + item.name = m.group(1) + item.url = response.url + m = self.price_re.search(response.body.to_string()) + if m: + item.price = m.group(1) + return [item] + + +SPIDER = TestSpider() diff --git a/scrapy/trunk/scrapy/tests/test_stats.py b/scrapy/trunk/scrapy/tests/test_stats.py new file mode 100644 index 000000000..3b7c2ac2d --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_stats.py @@ -0,0 +1,52 @@ +from unittest import TestCase, main +from scrapy.conf import settings +from scrapy.stats.statscollector import StatsCollector + +class StatsTest(TestCase): + def test_stats(self): + stats = StatsCollector(enabled=False) + self.assertEqual(stats, {}) + self.assertEqual(stats.getpath('anything'), None) + self.assertEqual(stats.getpath('anything', 'default'), 'default') + stats.setpath('test', 'value') + self.assertEqual(stats, {}) + + stats = StatsCollector(enabled=True) + self.assertEqual(stats, {}) + self.assertEqual(stats.getpath('anything'), None) + self.assertEqual(stats.getpath('anything', 'default'), 'default') + stats.setpath('test', 'value') + self.assertEqual(stats, {'test': 'value'}) + stats.setpath('test2', 23) + self.assertEqual(stats, {'test': 'value', 'test2': 23}) + stats.setpath('one/two', 'val2') + self.assertEqual(stats, {'test': 'value', 'test2': 23, 'one': {'two': 'val2'}}) + self.assertEqual(stats.getpath('one/two'), 'val2') + self.assertEqual(stats.getpath('one/three'), None) + self.assertEqual(stats.getpath('one/three', 'four'), 'four') + self.assertEqual(stats.getpath('one'), {'two': 'val2'}) + + # nodes must contain either data or other nodes, but not both! + self.assertRaises(TypeError, stats.setpath, 'one/two/three', 22) + + stats.delpath('test2') + self.assertEqual(stats, {'test': 'value', 'one': {'two': 'val2'}}) + stats.delpath('one/two') + self.assertEqual(stats, {'test': 'value', 'one': {}}) + stats.delpath('one') + self.assertEqual(stats, {'test': 'value'}) + + stats.setpath('one/other/three', 20) + self.assertEqual(stats.getpath('one/other/three'), 20) + stats.incpath('one/other/three') + self.assertEqual(stats.getpath('one/other/three'), 21) + stats.incpath('one/other/three', 4) + self.assertEqual(stats.getpath('one/other/three'), 25) + + stats.incpath('one/newnode', 1) + self.assertEqual(stats.getpath('one/newnode'), 1) + stats.incpath('one/newnode', -1) + self.assertEqual(stats.getpath('one/newnode'), 0) + +if __name__ == "__main__": + main() diff --git a/scrapy/trunk/scrapy/tests/test_storedb.py b/scrapy/trunk/scrapy/tests/test_storedb.py new file mode 100644 index 000000000..1a31dd1b7 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_storedb.py @@ -0,0 +1,74 @@ +from datetime import datetime, timedelta +from time import sleep + +from twisted.trial import unittest + + +# This is the DB used for testing +TEST_DB = "mysql://root:r00tpass@localhost/scrapingtest" + +class ProductComparisonTestCase(unittest.TestCase): + """ Test product comparison functions """ + + def setUp(self): + try: + import MySQLdb + from scrapy.utils.db import mysql_connect + mysql_connect(TEST_DB) + except ImportError: + raise unittest.SkipTest("MySQLdb module not available") + except MySQLdb.OperationalError: + raise unittest.SkipTest("Test database not available at: %s" % TEST_DB) + + def test_domaindatahistory(self): + from scrapy.store.db import DomainDataHistory + from time import sleep + + ddh = DomainDataHistory(TEST_DB, 'domain_data_history') + c = ddh.mysql_conn.cursor() + c.execute("DELETE FROM domain_data_history") + ddh.mysql_conn.commit() + + def now_nomicro(): + now = datetime.now() + return now - timedelta(microseconds=now.microsecond) + + assert hasattr(ddh.get('scrapy.org'), '__iter__') + self.assertEqual(list(ddh.get('scrapy.org')), []) + + self.assertEqual(ddh.domain_count(), 0) + + now = now_nomicro() + ddh.put('scrapy.org', 'value', timestamp=now) + self.assertEqual(list(ddh.get('scrapy.org')), [(now, 'value')]) + self.assertEqual(list(ddh.get('scrapy2.org')), []) + + sleep(1) + now2 = now_nomicro() + ddh.put('scrapy.org', 'newvalue', timestamp=now2) + self.assertEqual(list(ddh.getall('scrapy.org')), [(now2, 'newvalue'), (now, 'value')]) + + self.assertEqual(ddh.getlast('scrapy.org'), (now2, 'newvalue')) + self.assertEqual(ddh.getlast('scrapy.org', offset=1), (now, 'value')) + self.assertEqual(ddh.getlast('scrapy2.org'), None) + + ddh.remove('scrapy.org') + self.assertEqual(list(ddh.get('scrapy.org')), []) + + now3 = now_nomicro() + d1 = {'name': 'John', 'surname': 'Doe'} + ddh.put('scrapy.org', d1, timestamp=now3) + self.assertEqual(list(ddh.getall('scrapy.org')), [(now3, d1)]) + + # get path support + self.assertEqual(ddh.getlast('scrapy.org', path='name'), (now3, 'John')) + # behaviour for non existant paths + self.assertEqual(ddh.getlast('scrapy.org', path='name2'), (now3, None)) + + self.assertEqual(ddh.domain_count(), 1) + + self.assertEqual(list(ddh.getlast_alldomains()), + [('scrapy.org', now3, {'surname': 'Doe', 'name': 'John'})]) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_text_extraction.py b/scrapy/trunk/scrapy/tests/test_text_extraction.py new file mode 100644 index 000000000..544cdece7 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_text_extraction.py @@ -0,0 +1,111 @@ +# -*- coding: utf8 -*- +import unittest +from decimal import Decimal + +from decobot.utils.text_extraction import clean_markup, unquote_html, read_decimal, read_price + +class TextExtractionTest(unittest.TestCase): + def test_clean_markup(self): + + #with html tags + + html = u" <p> \xa0 <!-- comment --> <span>small</span> test </p> " + + self.assertEqual(clean_markup(html), + u'small test') + self.assertEqual(clean_markup(html, remove_tags=False, remove_root=False, remove_spaces=False, strip=False), + u' <p> \xa0 <!-- comment --> <span>small</span> test </p> ') + self.assertEqual(clean_markup(html, remove_tags=False, remove_root=True, strip=False), + u' <!-- comment --> <span>small</span> test ') + self.assertEqual(clean_markup(html, remove_tags=False, remove_root=False, remove_spaces=False, strip=True), + u'<p> \xa0 <!-- comment --> <span>small</span> test </p>') + self.assertEqual(clean_markup(html, remove_tags=False, remove_spaces=True, strip=False), + u' <!-- comment --> <span>small</span> test ') + self.assertEqual(clean_markup(html, remove_tags=True, remove_spaces=False, strip=False), + u' \xa0 small test ') + self.assertEqual(clean_markup(html, remove_tags=True, remove_spaces=True, strip=False), + u' small test ') + self.assertEqual(clean_markup(html, remove_tags=True, remove_root=True, remove_spaces=True, strip=True), + u'small test') + + #with xml tags + + self.assertEqual(clean_markup(u'<description/>', xml_doc=True), + u'') + + self.assertEqual(clean_markup(u'<description/>', xml_doc=True, remove_tags=False), + u'<description/>') + + self.assertEqual(clean_markup(u'<value>http://stephen.digivate2:8080/uploads/suppliers/4/11931_Picture 001.jpg</value>', xml_doc=True), + u'http://stephen.digivate2:8080/uploads/suppliers/4/11931_Picture 001.jpg') + self.assertEqual(clean_markup(u'<material> wood </material>', xml_doc=True), + u'wood') + self.assertEqual(clean_markup(u'<material> wood </material>', strip=False, xml_doc=True), + u' wood ') + self.assertEqual(clean_markup(u'<material><value>wood</value><value>iron</value></material>', xml_doc=True), + u'wood iron') + + self.assertEqual(clean_markup(u'<data><description>hi</description><![CDATA[This is a <tag> inside cdata </tag> <g>]]><nothing/><![CDATA[Another CDATA with tags <inside>]]></data>', xml_doc=True), + u'hi This is a inside cdata Another CDATA with tags') + + self.assertEqual(clean_markup(u'<data><description>hi</description><![CDATA[This is a <tag> inside cdata </tag> <g>]]><nothing/><![CDATA[Another CDATA with tags <inside>]]></data>', xml_doc=True, remove_tags=False), + u'<description>hi</description>This is a <tag> inside cdata </tag> <g><nothing/>Another CDATA with tags <inside>') + + self.assertEqual(clean_markup(u'<data><description>hi</description><![CDATA[This is a <tag> inside cdata </tag> <g>]]><nothing/><![CDATA[Another CDATA with tags <inside>]]></data>', remove_cdata=False, xml_doc=True,remove_tags=False), + u'<description>hi</description><![CDATA[This is a <tag> inside cdata </tag> <g>]]><nothing/><![CDATA[Another CDATA with tags <inside>]]>') + + self.assertEqual(clean_markup(u'<data><description>hi</description><![CDATA[This is a <tag> inside cdata </tag> <g>]]><nothing/><![CDATA[Another CDATA with tags <inside>]]></data>', remove_cdata=False, xml_doc=True), + u'hi <![CDATA[This is a <tag> inside cdata </tag> <g>]]> <![CDATA[Another CDATA with tags <inside>]]>') + + self.assertEqual(clean_markup(u'<![CDATA[This is a <tag> inside cdata </tag> <g>]]>', remove_cdata=False, xml_doc=True), + u'<![CDATA[This is a <tag> inside cdata </tag> <g>]]>') + + def test_unquote_html(self): + self.assertEqual(unquote_html(u'As low as £534,456.34!'), + 'As low as £534,456.34!') + self.assertEqual(unquote_html('As low as £534,456.34!'), + 'As low as £534,456.34!') + + def test_read_decimal(self): + self.assertEqual(read_decimal('asdf 234,234.45sdf '), + Decimal("234234.45")) + self.assertEqual(read_decimal('asdf 2234 sdf '), + Decimal("2234")) + self.assertEqual(read_decimal('947'), + Decimal("947")) + self.assertEqual(read_decimal('adsfg'), + None) + self.assertEqual(read_decimal('''stained, linseed oil finish, clear glas doors'''), + None) + + def test_read_price(self): + self.assertEqual(read_price(u'£549.97'), + Decimal("549.97")) + self.assertEqual(read_price(u'£5,499.97'), + Decimal("5499.97")) + self.assertEqual(read_price(u'Now: £5,499.97 for the last time'), + Decimal("5499.97")) + self.assertEqual(read_price(u'As low as 534,456.34!'), + Decimal("534456.34")) + self.assertEqual(read_price(u'As low as £534,456.34!'), + Decimal("534456.34")) + self.assertEqual(read_price(u'As low as £534,456.34!'), + Decimal("534456.34")) + self.assertEqual(read_price('asdf asdfg asdfg '), + None) + self.assertEqual(read_price('wrweq qewr -£12', True), + Decimal("-12")) + self.assertEqual(read_price('wrweq qewr £12', True), + Decimal("12")) + self.assertEqual(read_price('qwe ewqeq -12', True), + Decimal("0")) + self.assertEqual(read_price('qwe ewqeq 12', True), + Decimal("0")) + self.assertEqual(read_price('qwe ewqeq 12'), + Decimal("12")) + self.assertEqual(read_price('117.54 £'), + Decimal("117.54")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_utils_datatypes.py b/scrapy/trunk/scrapy/tests/test_utils_datatypes.py new file mode 100644 index 000000000..629eaf2ff --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_utils_datatypes.py @@ -0,0 +1,26 @@ +import unittest +from scrapy.utils.datatypes import PriorityQueue + +class DatatypesTestCase(unittest.TestCase): + + def test_priority_queue(self): + pq = PriorityQueue() + + pq.put('b', priority=1) + pq.put('a', priority=1) + pq.put('c', priority=1) + pq.put('z', priority=0) + pq.put('d', priority=2) + + v = [] + p = [] + while not pq.empty(): + priority, value = pq.get() + v.append(value) + p.append(priority) + + self.assertEqual(v, ['z', 'b', 'a', 'c', 'd']) + self.assertEqual(p, [0, 1, 1, 1, 2]) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py new file mode 100644 index 000000000..9b91c18da --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -0,0 +1,48 @@ +import unittest +from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url + +class UrlUtilsTest(unittest.TestCase): + + def test_url_is_from_any_domain(self): + url = 'http://www.wheele-bin-art.co.uk/get/product/123' + self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) + self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) + + url = 'http://wheele-bin-art.co.uk/get/product/123' + self.assertTrue(url_is_from_any_domain(url, ['wheele-bin-art.co.uk'])) + self.assertFalse(url_is_from_any_domain(url, ['art.co.uk'])) + + url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29' + self.assertFalse(url_is_from_any_domain(url, ['testdomain.com'])) + self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com'])) + + def test_safe_url_string(self): + # Motoko Kusanagi (Cyborg from Ghost in the Shell) + motoko = u'\u8349\u8599 \u7d20\u5b50' + self.assertEqual(safe_url_string(motoko), # note the %20 for space + '%E8%8D%89%E8%96%99%20%E7%B4%A0%E5%AD%90') + self.assertEqual(safe_url_string(motoko), + safe_url_string(safe_url_string(motoko))) + self.assertEqual(safe_url_string(u'\xa9'), # copyright symbol + '%C2%A9') + self.assertEqual(safe_url_string(u'\xa9', 'iso-8859-1'), + '%A9') + self.assertEqual(safe_url_string("http://www.scrapy.org/"), + 'http://www.scrapy.org/') + + alessi = u'/ecommerce/oggetto/Te \xf2/tea-strainer/1273' + + self.assertEqual(safe_url_string(alessi), + '/ecommerce/oggetto/Te%20%C3%B2/tea-strainer/1273') + + def test_safe_download_url(self): + self.assertEqual(safe_download_url('http://www.scrapy.org/../'), + 'http://www.scrapy.org/') + self.assertEqual(safe_download_url('http://www.scrapy.org/../../images/../image'), + 'http://www.scrapy.org/image') + self.assertEqual(safe_download_url('http://www.scrapy.org/dir/'), + 'http://www.scrapy.org/dir/') + +if __name__ == "__main__": + unittest.main() + diff --git a/scrapy/trunk/scrapy/tests/test_xpath.py b/scrapy/trunk/scrapy/tests/test_xpath.py new file mode 100644 index 000000000..481ea4293 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_xpath.py @@ -0,0 +1,175 @@ +import os +import re +import unittest + +import libxml2 + +from scrapy.http import Response +from scrapy.xpath.selector import XPathSelector +from scrapy.xpath.constructors import xmlDoc_from_xml +from scrapy.xpath.iterator import XMLNodeIterator + +class XPathTestCase(unittest.TestCase): + + def setUp(self): + libxml2.debugMemory(1) + + def tearDown(self): + libxml2.cleanupParser() + leaked_bytes = libxml2.debugMemory(0) + assert leaked_bytes == 0, "libxml2 memory leak detected: %d bytes" % leaked_bytes + + def test_selector_simple(self): + """Simple selector tests""" + body = "<p><input name='a'value='1'/><input name='b'value='2'/></p>" + response = Response(domain="example.com", url="http://example.com", body=body) + xpath = XPathSelector(response) + + xl = xpath.x('//input') + self.assertEqual(2, len(xl)) + for x in xl: + assert isinstance(x, XPathSelector) + + self.assertEqual(xpath.x('//input').extract(), + [x.extract() for x in xpath.x('//input')]) + + self.assertEqual([x.extract() for x in xpath.x("//input[@name='a']/@name")], + [u'a']) + self.assertEqual([x.extract() for x in xpath.x("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + [u'12.0']) + + self.assertEqual(xpath.x("concat('xpath', 'rules')").extract(), + [u'xpathrules']) + self.assertEqual([x.extract() for x in xpath.x("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + [u'12']) + + def test_selector_nested(self): + """Nested selector tests""" + body = """<body> + <div class='one'> + <ul> + <li>one</li><li>two</li> + </ul> + </div> + <div class='two'> + <ul> + <li>four</li><li>five</li><li>six</li> + </ul> + </div> + </body>""" + + response = Response(domain="example.com", url="http://example.com", body=body) + x = XPathSelector(response) + + divtwo = x.x('//div[@class="two"]') + self.assertEqual(divtwo.x("//li").extract(), + ["<li>one</li>", "<li>two</li>", "<li>four</li>", "<li>five</li>", "<li>six</li>"]) + self.assertEqual(divtwo.x("./ul/li").extract(), + ["<li>four</li>", "<li>five</li>", "<li>six</li>"]) + self.assertEqual(divtwo.x(".//li").extract(), + ["<li>four</li>", "<li>five</li>", "<li>six</li>"]) + self.assertEqual(divtwo.x("./li").extract(), + []) + + def test_selector_re(self): + body = """<div>Name: Mary + <ul> + <li>Name: John</li> + <li>Age: 10</li> + <li>Name: Paul</li> + <li>Age: 20</li> + </ul> + Age: 20 + </div> + + """ + response = Response(domain="example.com", url="http://example.com", body=body) + x = XPathSelector(response) + + name_re = re.compile("Name: (\w+)") + self.assertEqual(x.x("//ul/li").re(name_re), + ["John", "Paul"]) + self.assertEqual(x.x("//ul/li").re("Age: (\d+)"), + ["10", "20"]) + + def test_selector_namespaces_simple(self): + body = """ + <test xmlns:somens="http://scrapy.org"> + <somens:a id="foo"/> + <a id="bar">found</a> + </test> + """ + + response = Response(domain="example.com", url="http://example.com", body=body) + x = XPathSelector(response, constructor=xmlDoc_from_xml) + + x.register_namespace("somens", "http://scrapy.org") + self.assertEqual(x.x("//somens:a").extract(), + ['<somens:a id="foo"/>']) + + + def test_selector_namespaces_multiple(self): + body = """<?xml version="1.0" encoding="UTF-8"?> +<BrowseNode xmlns="http://webservices.amazon.com/AWSECommerceService/2005-10-05" + xmlns:b="http://somens.com" + xmlns:p="http://www.scrapy.org/product" > + <b:Operation>hello</b:Operation> + <TestTag b:att="value"><Other>value</Other></TestTag> + <p:SecondTestTag><material/><price>90</price><p:name>Dried Rose</p:name></p:SecondTestTag> +</BrowseNode> + """ + response = Response(domain="example.com", url="http://example.com", body=body) + x = XPathSelector(response, constructor=xmlDoc_from_xml) + + x.register_namespace("xmlns", "http://webservices.amazon.com/AWSECommerceService/2005-10-05") + x.register_namespace("p", "http://www.scrapy.org/product") + x.register_namespace("b", "http://somens.com") + self.assertEqual(len(x.x("//xmlns:TestTag")), 1) + self.assertEqual(x.x("//b:Operation/text()").extract()[0], 'hello') + self.assertEqual(x.x("//xmlns:TestTag/@b:att").extract()[0], 'value') + self.assertEqual(x.x("//p:SecondTestTag/xmlns:price/text()").extract()[0], '90') + self.assertEqual(x.x("//p:SecondTestTag").x("./xmlns:price/text()")[0].extract(), '90') + self.assertEqual(x.x("//p:SecondTestTag/xmlns:material").extract()[0], '<material/>') + + def test_http_header_encoding_precedence(self): + # u'\xa3' = pound symbol in unicode + # u'\xc2\xa3' = pound symbol in utf-8 + # u'\xa3' = pound symbol in latin-1 (iso-8859-1) + + meta = u'<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' + head = u'<head>' + meta + u'</head>' + body_content = u'<span id="blank">\xa3</span>' + body = u'<body>' + body_content + u'</body>' + html = u'<html>' + head + body + u'</html>' + encoding = 'utf-8' + html_utf8 = html.encode(encoding) + + headers = {'Content-Type': ['text/html; charset=utf-8']} + response = Response(domain="example.com", url="http://example.com", headers=headers, body=html_utf8) + x = XPathSelector(response) + self.assertEquals(x.x("//span[@id='blank']/text()").extract(), + [u'\xa3']) + + def test_iterator(self): + body = """<?xml version="1.0" encoding="UTF-8"?> +<products xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="someschmea.xsd"> + <product id="001"> + <type>Type 1</type> + <name>Name 1</name> + </product> + <product id="002"> + <type>Type 2</type> + <name>Name 2</name> + </product> +</products> + """ + response = Response(domain="example.com", url="http://example.com", body=body) + attrs = [] + for x in XMLNodeIterator(response, 'product'): + attrs.append((x.x("@id").extract(), x.x("name/text()").extract(), x.x("./type/text()").extract())) + + self.assertEqual(attrs, + [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/utils/__init__.py b/scrapy/trunk/scrapy/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapy/trunk/scrapy/utils/c14n.py b/scrapy/trunk/scrapy/utils/c14n.py new file mode 100644 index 000000000..88cdd16df --- /dev/null +++ b/scrapy/trunk/scrapy/utils/c14n.py @@ -0,0 +1,110 @@ +""" +Canonicalization routine for URLs +""" +import re +import urlparse + +_hexmatcher = re.compile("\%[0-9A-Fa-f]{2}") + +def get_canonized_port(port): + if port and port != 80: + cport = ":" + str(port) + else: + cport = "" + return cport + +def create_params_dict(params): + param_dict = {} + for param in params.split("&"): + nvpair = param.split("=") + if len(nvpair) >= 2: + param_dict[nvpair[0]] = '='.join(nvpair[1:]) + else: + param_dict[nvpair[0]] = None + return param_dict + +def create_unique_params_list(params): + """ + This function build a query string from unique params only. + Unique params are params with unique key=>value pair. So param=qwe and param=123 + are diffrent and a query string will be param=qwe¶m=123. + If input params will contain pairs like this param=123 and param=123 a query + string will be just param=123 + We have to do this because some ASP sites uses the same parameters names in the URLs, + but with diffrent values. Such queries are interpreted as arrays in the ASP framework. + """ + param_list = [] + if params: + for param in params.split("&"): + nvpair = param.split("=", 1) + if len(nvpair) >= 2: + param_list.append(nvpair[0] + '=' + nvpair[1]) + else: + param_list.append(nvpair[0] + '=') + if param_list: + p_list = [] + # select only unique pairs, it is possible to use set() for this but change order of pairs... + for param in param_list: + if param not in p_list: + p_list.append(param) + return '&'.join(p_list) + return '' + +def convert_escaped(string_to_convert): + hexvalues = _hexmatcher.findall(string_to_convert) + for hexvalue in hexvalues: + string_to_convert = string_to_convert.replace( + hexvalue, chr(int(hexvalue[1:len(hexvalue)],16))) + return string_to_convert + +def remove_dirs(path): + dirs = path.split("/") + dirs.reverse() + stripped_dirs = [] + ellipsis = 0 + for dir in dirs: + if dir == ".." : + ellipsis += 1 + elif dir == "." : + pass + else: + if ellipsis > 0 : + ellipsis -= 1 + else: + stripped_dirs.append(dir) + stripped_dirs.reverse() + return "/".join(stripped_dirs) + +def canonicalize(url): + """ + This method c14ns the url we are passed by doing the following + 1) lowercasing the hostname and scheme + 2) removing the port if it is 80 + 3) making the query params unique and alphabetized + 4) removing url encoding + 5) Removing any ./ + 6) Removing and ../ and their preceding directories + """ + + # Remove the fragment marker, even if it's url encoded + defragmented_url = urlparse.urldefrag(url)[0] + + # This automatically converts the scheme and hostname to lower case + parsed = urlparse.urlparse(defragmented_url) + + # Get the port in the correct format + cport = get_canonized_port(parsed.port) + + query = create_unique_params_list(parsed.query) + + # Remove the url encoded params from the path and query string + query = convert_escaped(query) + if query : + query = "?" + query + path = convert_escaped(parsed.path) + + # Remove the ../ and the previous directories and remove the ./ + path = remove_dirs(path) + + # Now put it all back together + return "%s://%s%s%s%s" % (parsed.scheme, parsed.hostname, cport, path, query) diff --git a/scrapy/trunk/scrapy/utils/datatypes.py b/scrapy/trunk/scrapy/utils/datatypes.py new file mode 100644 index 000000000..ab32a82ce --- /dev/null +++ b/scrapy/trunk/scrapy/utils/datatypes.py @@ -0,0 +1,458 @@ +""" +This module contains data types used by Scrapy which are not included in the +Python Standard Library. + +This module must not depend on any module outside the Standard Library. +""" + +import copy +import gzip +import Queue +import bisect +from cStringIO import StringIO + +class MergeDict(object): + """ + A simple class for creating new "virtual" dictionaries that actualy look + up values in more than one dictionary, passed in the constructor. + """ + def __init__(self, *dicts): + self.dicts = dicts + + def __getitem__(self, key): + for dict_ in self.dicts: + try: + return dict_[key] + except KeyError: + pass + raise KeyError + + def __copy__(self): + return self.__class__(*self.dicts) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + def getlist(self, key): + for dict in self.dicts: + try: + return dict.getlist(key) + except KeyError: + pass + raise KeyError + + def items(self): + item_list = [] + for dict in self.dicts: + item_list.extend(dict.items()) + return item_list + + def has_key(self, key): + for dict in self.dicts: + if key in dict: + return True + return False + + __contains__ = has_key + + def copy(self): + """ returns a copy of this object""" + return self.__copy__() + +class SortedDict(dict): + "A dictionary that keeps its keys in the order in which they're inserted." + def __init__(self, data=None): + if data is None: + data = {} + dict.__init__(self, data) + self.keyOrder = data.keys() + + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + if key not in self.keyOrder: + self.keyOrder.append(key) + + def __delitem__(self, key): + dict.__delitem__(self, key) + self.keyOrder.remove(key) + + def __iter__(self): + for k in self.keyOrder: + yield k + + def items(self): + return zip(self.keyOrder, self.values()) + + def iteritems(self): + for key in self.keyOrder: + yield key, dict.__getitem__(self, key) + + def keys(self): + return self.keyOrder[:] + + def iterkeys(self): + return iter(self.keyOrder) + + def values(self): + return [dict.__getitem__(self, k) for k in self.keyOrder] + + def itervalues(self): + for key in self.keyOrder: + yield dict.__getitem__(self, key) + + def update(self, dict): + for k, v in dict.items(): + self.__setitem__(k, v) + + def setdefault(self, key, default): + if key not in self.keyOrder: + self.keyOrder.append(key) + return dict.setdefault(self, key, default) + + def value_for_index(self, index): + "Returns the value of the item at the given zero-based index." + return self[self.keyOrder[index]] + + def insert(self, index, key, value): + "Inserts the key, value pair before the item with the given index." + if key in self.keyOrder: + n = self.keyOrder.index(key) + del self.keyOrder[n] + if n < index: + index -= 1 + self.keyOrder.insert(index, key) + dict.__setitem__(self, key, value) + + def copy(self): + "Returns a copy of this object." + # This way of initializing the copy means it works for subclasses, too. + obj = self.__class__(self) + obj.keyOrder = self.keyOrder + return obj + + def __repr__(self): + """ + Replaces the normal dict.__repr__ with a version that returns the keys + in their sorted order. + """ + return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) + +class MultiValueDictKeyError(KeyError): + pass + +class MultiValueDict(dict): + """ + A subclass of dictionary customized to handle multiple values for the same key. + + >>> d = MultiValueDict({'name': ['Adrian', 'Simon'], 'position': ['Developer']}) + >>> d['name'] + 'Simon' + >>> d.getlist('name') + ['Adrian', 'Simon'] + >>> d.get('lastname', 'nonexistent') + 'nonexistent' + >>> d.setlist('lastname', ['Holovaty', 'Willison']) + + This class exists to solve the irritating problem raised by cgi.parse_qs, + which returns a list for every key, even though most Web forms submit + single name-value pairs. + """ + def __init__(self, key_to_list_mapping=()): + dict.__init__(self, key_to_list_mapping) + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, dict.__repr__(self)) + + def __getitem__(self, key): + """ + Returns the last data value for this key, or [] if it's an empty list; + raises KeyError if not found. + """ + try: + list_ = dict.__getitem__(self, key) + except KeyError: + raise MultiValueDictKeyError, "Key %r not found in %r" % (key, self) + try: + return list_[-1] + except IndexError: + return [] + + def __setitem__(self, key, value): + dict.__setitem__(self, key, [value]) + + def __copy__(self): + return self.__class__(dict.items(self)) + + def __deepcopy__(self, memo=None): + if memo is None: + memo = {} + result = self.__class__() + memo[id(self)] = result + for key, value in dict.items(self): + dict.__setitem__(result, copy.deepcopy(key, memo), copy.deepcopy(value, memo)) + return result + + def get(self, key, default=None): + "Returns the default value if the requested data doesn't exist" + try: + val = self[key] + except KeyError: + return default + if val == []: + return default + return val + + def getlist(self, key): + "Returns an empty list if the requested data doesn't exist" + try: + return dict.__getitem__(self, key) + except KeyError: + return [] + + def setlist(self, key, list_): + dict.__setitem__(self, key, list_) + + def setdefault(self, key, default=None): + if key not in self: + self[key] = default + return self[key] + + def setlistdefault(self, key, default_list=()): + if key not in self: + self.setlist(key, default_list) + return self.getlist(key) + + def appendlist(self, key, value): + "Appends an item to the internal list associated with key" + self.setlistdefault(key, []) + dict.__setitem__(self, key, self.getlist(key) + [value]) + + def items(self): + """ + Returns a list of (key, value) pairs, where value is the last item in + the list associated with the key. + """ + return [(key, self[key]) for key in self.keys()] + + def lists(self): + "Returns a list of (key, list) pairs." + return dict.items(self) + + def values(self): + "Returns a list of the last value on every key list." + return [self[key] for key in self.keys()] + + def copy(self): + "Returns a copy of this object." + return self.__deepcopy__() + + def update(self, *args, **kwargs): + "update() extends rather than replaces existing key lists. Also accepts keyword args." + if len(args) > 1: + raise TypeError, "update expected at most 1 arguments, got %d" % len(args) + if args: + other_dict = args[0] + if isinstance(other_dict, MultiValueDict): + for key, value_list in other_dict.lists(): + self.setlistdefault(key, []).extend(value_list) + else: + try: + for key, value in other_dict.items(): + self.setlistdefault(key, []).append(value) + except TypeError: + raise ValueError, "MultiValueDict.update() takes either a MultiValueDict or dictionary" + for key, value in kwargs.iteritems(): + self.setlistdefault(key, []).append(value) + +class DotExpandedDict(dict): + """ + A special dictionary constructor that takes a dictionary in which the keys + may contain dots to specify inner dictionaries. It's confusing, but this + example should make sense. + + >>> d = DotExpandedDict({'person.1.firstname': ['Simon'], \ + 'person.1.lastname': ['Willison'], \ + 'person.2.firstname': ['Adrian'], \ + 'person.2.lastname': ['Holovaty']}) + >>> d + {'person': {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}}} + >>> d['person'] + {'1': {'lastname': ['Willison'], 'firstname': ['Simon']}, '2': {'lastname': ['Holovaty'], 'firstname': ['Adrian']}} + >>> d['person']['1'] + {'lastname': ['Willison'], 'firstname': ['Simon']} + + # Gotcha: Results are unpredictable if the dots are "uneven": + >>> DotExpandedDict({'c.1': 2, 'c.2': 3, 'c': 1}) + {'c': 1} + """ + def __init__(self, key_to_list_mapping): + for k, v in key_to_list_mapping.items(): + current = self + bits = k.split('.') + for bit in bits[:-1]: + current = current.setdefault(bit, {}) + # Now assign value to current position + try: + current[bits[-1]] = v + except TypeError: # Special-case if current isn't a dict. + current = {bits[-1] : v} + +class FileDict(dict): + """ + A dictionary used to hold uploaded file contents. The only special feature + here is that repr() of this object won't dump the entire contents of the + file to the output. A handy safeguard for a large file upload. + """ + def __repr__(self): + if 'content' in self: + d = dict(self, content='<omitted>') + return dict.__repr__(d) + return dict.__repr__(self) + +class Sitemap(object): + """Sitemap class is used to build a map of the traversed pages""" + + def __init__(self): + self._nodes = {} + self._roots = [] + + def add_node(self, url, parent_url): + if not url in self._nodes: + parent = self._nodes.get(parent_url, None) + node = SiteNode(url) + self._nodes[url] = node + if parent: + parent.add_child(node) + else: + self._roots.append(node) + + def add_item(self, url, item): + if url in self._nodes: + self._nodes[url].itemnames.append(str(item)) + + def to_string(self): + s = ''.join([n.to_string(0) for n in self._roots]) + return s + +class SiteNode(object): + """Class to represent a site node (page, image or any other file)""" + + def __init__(self, url): + self.url = url + self.itemnames = [] + self.children = [] + self.parent = None + + def add_child(self, node): + self.children.append(node) + node.parent = self + + def to_string(self, level=0): + s = "%s%s\n" % (' '*level, self.url) + if self.itemnames: + for n in self.itemnames: + s += "%sScraped: %s\n" % (' '*(level+1), n) + for node in self.children: + s += node.to_string(level+1) + return s + +class CaselessDict(dict): + def __init__(self, other=None): + if other: + # Doesn't do keyword args + if isinstance(other, dict): + for k, v in other.items(): + dict.__setitem__(self, self.normkey(k), v) + else: + for k, v in other: + dict.__setitem__(self, self.normkey(k), v) + + def __getitem__(self, key): + return dict.__getitem__(self, self.normkey(key)) + + def __setitem__(self, key, value): + dict.__setitem__(self, self.normkey(key), value) + + def __delitem__(self, key): + dict.__delitem__(self, self.normkey(key)) + + def __contains__(self, key): + return dict.__contains__(self, self.normkey(key)) + + def normkey(self, key): + return key.lower() + + def has_key(self, key): + return dict.has_key(self, self.normkey(key)) + + def get(self, key, def_val=None): + return dict.get(self, self.normkey(key), def_val) + + def setdefault(self, key, def_val=None): + return dict.setdefault(self, self.normkey(key), def_val) + + def update(self, other): + for k, v in other.items(): + dict.__setitem__(self, self.normkey(k), v) + + def fromkeys(self, iterable, value=None): + d = CaselessDict() + for k in iterable: + dict.__setitem__(d, self.normkey(k), value) + return d + + def pop(self, key, def_val=None): + return dict.pop(self, self.normkey(key), def_val) + + +class PriorityQueue(Queue.Queue): + + def __len__(self): + return self.qsize() + + def _init(self, maxsize): + self.maxsize = maxsize + # Python 2.5 uses collections.deque, but we can't because + # we need insert(pos, item) for our priority stuff + self.queue = [] + + def put(self, item, priority=0, block=True, timeout=None): + """Puts an item onto the queue with a numeric priority (default is zero). + + Note that we are "shadowing" the original Queue.Queue put() method here. + """ + Queue.Queue.put(self, (priority, item), block, timeout) + + def _put(self, item): + """Override of the Queue._put to support prioritisation.""" + # Priorities must be integers! + priority = int(item[0]) + + # Using a tuple (priority+1,) finds us the correct insertion + # position to maintain the existing ordering. + self.queue.insert(bisect.bisect_left(self.queue, (priority+1,)), item) + + def _get(self): + """Override of Queue._get(). Strips the priority.""" + return self.queue.pop(0) + + +class PriorityStack(PriorityQueue): + def _put(self, item): + priority = int(item[0]) + self.queue.insert(bisect.bisect_left(self.queue, (priority,)), item) + +class gzStringIO: + """a file like object, similar to StringIO, but gzip-compressed.""" + def __init__(self, data, compress_level = 9, filename = ""): + self._s = StringIO() + g = gzip.GzipFile(filename, "wb", compress_level, self._s) + g.write(data) + g.flush() + g.close() + def read(self): + return self._s.getvalue() + diff --git a/scrapy/trunk/scrapy/utils/db.py b/scrapy/trunk/scrapy/utils/db.py new file mode 100644 index 000000000..274b0f4e7 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/db.py @@ -0,0 +1,22 @@ +""" +Function for dealing with databases +""" +import re + +def mysql_connect(db_uri, **kwargs): + """ + Connects to a MySQL DB given a mysql URI + """ + import MySQLdb + + if not db_uri or not db_uri.startswith('mysql://'): + raise Exception("Incorrect MySQL URI: %s" % db_uri) + m = re.search(r"mysql:\/\/(?P<user>[^:]+)(:(?P<passwd>[^@]+))?@(?P<host>[^/]+)/(?P<db>.*)$", db_uri) + if m: + d = m.groupdict() + if d['passwd'] is None: + del(d['passwd']) + + d['charset'] = "utf8" + d.update(kwargs) + return MySQLdb.connect(**d) diff --git a/scrapy/trunk/scrapy/utils/display.py b/scrapy/trunk/scrapy/utils/display.py new file mode 100644 index 000000000..710e7d743 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/display.py @@ -0,0 +1,49 @@ +""" +Helper functions for formatting and pretty printing some objects +""" +import sys +import pprint as pypprint + +from scrapy.item import ScrapedItem +from scrapy.http import Request, Response + +nocolour = False + +def colorize(text): + if nocolour or not sys.stdout.isatty(): + return text + try: + from pygments import highlight + from pygments.formatters import TerminalFormatter + from pygments.lexers import PythonLexer + return highlight(text, PythonLexer(), TerminalFormatter()) + except ImportError: + return text + +def pformat_dictobj(obj): + clsname = obj.__class__.__name__ + return "%s(%s)\n" % (clsname, colorize(pypprint.pformat(obj.__dict__))) + +def pprint_dictobj(obj): + print pformat_dictobj(obj) + +def pformat(obj, *args, **kwargs): + """ + Wrapper which autodetects the object type and uses the proper formatting + function + """ + + if isinstance(obj, (list, tuple)): + return "".join([pformat(i, *args, **kwargs) for i in obj]) + elif isinstance(obj, (ScrapedItem, Request, Response)): + return pformat_dictobj(obj) + else: + return colorize(pypprint.pformat(obj)) + +def pprint(obj, *args, **kwargs): + """ + Wrapper which autodetects the object type and uses the proper printing + function + """ + + print pformat(obj, *args, **kwargs) diff --git a/scrapy/trunk/scrapy/utils/misc.py b/scrapy/trunk/scrapy/utils/misc.py new file mode 100644 index 000000000..3be118b6a --- /dev/null +++ b/scrapy/trunk/scrapy/utils/misc.py @@ -0,0 +1,180 @@ +""" +Auxiliary functions which doesn't fit anywhere else +""" +import re + +from twisted.internet import defer, reactor +from twisted.python import failure + +from scrapy.core.exceptions import UsageError +from scrapy.utils.python import flatten +from decobot.utils.text_extraction import unquote_html + +def dict_updatedefault(D, E, **F): + """ + updatedefault(D, E, **F) -> None. + + Update D from E and F: for k in E: D.setdefault(k, E[k]) + (if E has keys else: for (k, v) in E: D.setdefault(k, v)) + then: for k in F: D.setdefault(k, F[k]) + """ + for k in E: + if isinstance(k, tuple): + k, v = k + else: + v = E[k] + D.setdefault(k, v) + + for k in F: + D.setdefault(k, F[k]) + + +def defer_fail(_failure): + """same as twsited.internet.defer.fail, but delay calling errback """ + d = defer.Deferred() + reactor.callLater(0, d.errback, _failure) + return d + + +def defer_succeed(result): + """same as twsited.internet.defer.succed, but delay calling callback""" + d = defer.Deferred() + reactor.callLater(0, d.callback, result) + return d + + +def mustbe_deferred(f, *args, **kw): + """same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback""" + deferred = None + try: + result = f(*args, **kw) + except: + return defer_fail(failure.Failure()) + else: + if isinstance(result, defer.Deferred): + return result + elif isinstance(result, failure.Failure): + return defer_fail(result) + else: + return defer_succeed(result) + return deferred + + +def chain_deferred(d1, d2): + if callable(d2): + d2 = lambda_deferred(d2) + + def _pause(_): + d2.pause() + reactor.callLater(0, d2.unpause) + return _ + + def _reclaim(_): + return d2 + + #d1.addBoth(_pause) ## needs more debugging before reenable it + d1.chainDeferred(d2) + d1.addBoth(_reclaim) + return d1 + + +def lambda_deferred(func): + deferred = defer.Deferred() + def _success(res): + d = func() + d.callback(res) + return d + def _fail(res): + d = func() + d.errback(res) + return d + return deferred.addCallbacks(_success, _fail) + + +def memoize(cache, hash): + def decorator(func): + def wrapper(*args, **kwargs): + key = hash(*args, **kwargs) + if key in cache: + return defer_succeed(cache[key]) + + def _store(_): + cache[key] = _ + return _ + + result = func(*args, **kwargs) + if isinstance(result, defer.Deferred): + return result.addBoth(_store) + cache[key] = result + return result + return wrapper + return decorator + + +def deferred_degenerate(generator): + generator = iter(generator) + deferred = defer.Deferred() + result = [] + def _next(): + try: + result.append(generator.next()) + except StopIteration: + reactor.callLater(0, deferred.callback, result) + except: + reactor.callLater(0, deferred.errback, failure.Failure()) + else: + reactor.callLater(0, _next) + _next() + return deferred + + +def stats_getpath(dict_, path, default=None): + for key in path.split('/'): + if key in dict_: + dict_ = dict_[key] + else: + return default + return dict_ + +def load_class(class_path): + """Load a class given its absolute class path, and return it without + instantiating it""" + try: + dot = class_path.rindex('.') + except ValueError: + raise UsageError, '%s isn\'t a module' % class_path + module, classname = class_path[:dot], class_path[dot+1:] + try: + mod = __import__(module, {}, {}, ['']) + except ImportError, e: + raise UsageError, 'Error importing %s: "%s"' % (module, e) + try: + cls = getattr(mod, classname) + except AttributeError: + raise UsageError, 'module "%s" does not define a "%s" class' % (module, classname) + + return cls + +def extract_regex(regex, text, encoding): + """Extract a list of unicode strings from the given text/encoding using the following policies: + + * if the regex contains a named group called "extract" that will be returned + * if the regex contains multiple numbered groups, all those will be returned (flattened) + * if the refex doesn't contain any group the entire regex matching is returned + """ + + if isinstance(regex, basestring): + regex = re.compile(regex) + + try: + strings = [regex.search(text).group('extract')] # named group + except: + strings = regex.findall(text) # full regex or numbered groups + strings = flatten(strings) + + if isinstance(text, unicode): + return [unquote_html(s, keep_reserved=True) for s in strings] + else: + return [unquote_html(unicode(s, encoding), keep_reserved=True) for s in strings] + + diff --git a/scrapy/trunk/scrapy/utils/python.py b/scrapy/trunk/scrapy/utils/python.py new file mode 100644 index 000000000..23e08075a --- /dev/null +++ b/scrapy/trunk/scrapy/utils/python.py @@ -0,0 +1,55 @@ +""" +This module contains essential stuff that should've come with Python itself ;) +""" + +from sgmllib import SGMLParser + +class FixedSGMLParser(SGMLParser): + """The SGMLParser that comes with Python has a bug in the convert_charref() + method. This is the same class with the bug fixed""" + + def convert_charref(self, name): + """This method fixes a bug in Python's SGMLParser.""" + try: + n = int(name) + except ValueError: + return + if not 0 <= n <= 127 : # ASCII ends at 127, not 255 + return + return self.convert_codepoint(n) + + +def flatten(x): + """flatten(sequence) -> list + + Returns a single, flat list which contains all elements retrieved + from the sequence and all recursively contained sub-sequences + (iterables). + + Examples: + >>> [1, 2, [3,4], (5,6)] + [1, 2, [3, 4], (5, 6)] + >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)]) + [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]""" + + result = [] + for el in x: + if hasattr(el, "__iter__"): + result.extend(flatten(el)) + else: + result.append(el) + return result + + +def unique(list_): + """efficient function to uniquify a list preserving item order""" + seen = {} + result = [] + for item in list_: + if item in seen: + continue + seen[item] = 1 + result.append(item) + return result + + diff --git a/scrapy/trunk/scrapy/utils/serialization.py b/scrapy/trunk/scrapy/utils/serialization.py new file mode 100644 index 000000000..82bd01eb3 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/serialization.py @@ -0,0 +1,74 @@ +""" +This module provides a universal API for different serialization formats. + +Keep in mind that not all formats support all types for serialization, and some +formats (like pprint) may be unsafe for unserialization (like pprint) but it +may still very convenient for serialization. +""" + +import datetime +import decimal +import cPickle as pickle +import pprint + +import simplejson + +class ScrapyJSONEncoder(simplejson.JSONEncoder): + """ + JSONEncoder subclass that knows how to encode date/time and decimal types. + """ + + DATE_FORMAT = "%Y-%m-%d" + TIME_FORMAT = "%H:%M:%S" + + def default(self, o): + if isinstance(o, datetime.datetime): + return o.strftime("%s %s" % (self.DATE_FORMAT, self.TIME_FORMAT)) + elif isinstance(o, datetime.date): + return o.strftime(self.DATE_FORMAT) + elif isinstance(o, datetime.time): + return o.strftime(self.TIME_FORMAT) + elif isinstance(o, decimal.Decimal): + return str(o) + else: + return super(ScrapyJSONEncoder, self).default(o) + +serialize_funcs = { + 'json': lambda obj: simplejson.dumps(obj, cls=ScrapyJSONEncoder), + 'pprint': lambda obj: pprint.pformat(obj), + 'pickle': lambda obj: pickle.dumps(obj), +} + +unserialize_funcs = { + 'json': lambda text: simplejson.loads(text), + 'pprint': lambda text: eval(text), + 'pickle': lambda text: pickle.loads(text), +} + +def serialize(obj, format='pickle'): + """ + Main entrance point for serialization. + + Supported formats: pickle, json, pprint + """ + try: + func = serialize_funcs[format] + return func(obj) + except KeyError: + raise TypeError("Unknown serialization format: %s" % format) + +def unserialize(text, format='pickle'): + """ + Main entrance point for unserialization. + """ + try: + func = unserialize_funcs[format] + return func(text) + except KeyError: + raise TypeError("Unknown serialization format: %s" % format) + +def parse_jsondatetime(text): + if isinstance(text, basestring): + return datetime.datetime.strptime(text, "%s %s" % \ + (ScrapyJSONEncoder.DATE_FORMAT, ScrapyJSONEncoder.TIME_FORMAT)) + diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py new file mode 100644 index 000000000..5e62fca17 --- /dev/null +++ b/scrapy/trunk/scrapy/utils/url.py @@ -0,0 +1,83 @@ +""" +This module contains general purpose URL functions not found in the standard +library. +""" + +import re +import urlparse +import urllib +import posixpath + +def url_is_from_any_domain(url, domains): + """Return True if the url belongs to the given domain""" + host = urlparse.urlparse(url).hostname + + if host: + return any(((host == d) or (host.endswith('.%s' % d)) for d in domains)) + else: + return False + +def url_is_from_spider(url, spider): + """Return True if the url belongs to the given spider""" + domains = [spider.domain_name] + spider.extra_domain_names + return url_is_from_any_domain(url, domains) + +def urljoin_rfc(base, ref): + """ + Fixed urlparse.urljoin version that handles + relative query string as RFC states. + """ + if ref.startswith('?'): + fpart = urlparse.urlsplit(str(base))[2].rsplit('/', 1)[-1] + ref = ''.join([fpart, ref]) + # convert ref to a string. This should already + # be the case, however, many spiders do not convert. + return urlparse.urljoin(base, str(ref)) + + +_reserved = ';/?:@&=+$|,#' # RFC 2396 (Generic Syntax) +_safe_chars = urllib.always_safe + '%' + _reserved + +def safe_url_string(url, use_encoding='utf8'): + """Convert a unicode (or utf8 string) object into a legal URL. + + Illegal characters are escaped. See rfc3968. + + It is safe to call this function multiple times. Do not pass this + function strings in encodings other than utf8. + + The use_encoding argument is the encoding to use to determine the numerical + values in the escaping. For urls on html pages, you should use the original + encoding of that page. + + html pages you should escape urls in the original encoding + of the page and not using utf8. + """ + s = url.encode(use_encoding) + return urllib.quote(s, _safe_chars) + + +_parent_dirs = re.compile(r'/?(\.\./)+') + +def safe_download_url(url): + """ Make a url for download. This will call safe_url_string + and then strip the fragment, if one exists. The path will + be normalised. + + If the path is outside the document root, it will be changed + to be within the document root. + """ + safe_url = safe_url_string(url) + scheme, netloc, path, query, fragment = urlparse.urlsplit(safe_url) + if path: + path = _parent_dirs.sub('', posixpath.normpath(path)) + if url.endswith('/') and not path.endswith('/'): + path += '/' + else: + path = '/' + return urlparse.urlunsplit((scheme, netloc, path, query, '')) + + +def is_url(text): + return text.partition("://")[0] in ('file', 'http', 'https') + diff --git a/scrapy/trunk/scrapy/xpath/__init__.py b/scrapy/trunk/scrapy/xpath/__init__.py new file mode 100644 index 000000000..a221e53a0 --- /dev/null +++ b/scrapy/trunk/scrapy/xpath/__init__.py @@ -0,0 +1,15 @@ +""" +The scrapy.xpath module provides useful classes for parsing HTML and XML +documents using XPath. It requires libxml2 and its python bindings. + +This parent module exports the classes most commonly used when building +spiders, for convenience. + +* XPath - a simple class to represent a XPath expression +* XPathSelector - to extract data using XPaths (parses the entire response) +* XMLNodeIterator - to iterate over XML nodes without parsing the entire response in memory +""" + +from scrapy.xpath.types import XPath +from scrapy.xpath.selector import XPathSelector +from scrapy.xpath.iterator import XMLNodeIterator diff --git a/scrapy/trunk/scrapy/xpath/constructors.py b/scrapy/trunk/scrapy/xpath/constructors.py new file mode 100644 index 000000000..95eedf5d8 --- /dev/null +++ b/scrapy/trunk/scrapy/xpath/constructors.py @@ -0,0 +1,28 @@ +""" +This module provides functions for generating libxml2 documents (xmlDoc). + +Constructors must receive a Response object and return a xmlDoc object. +""" + +import libxml2 + +xml_parser_options = libxml2.XML_PARSE_RECOVER + \ + libxml2.XML_PARSE_NOERROR + \ + libxml2.XML_PARSE_NOWARNING + +html_parser_options = libxml2.HTML_PARSE_RECOVER + \ + libxml2.HTML_PARSE_NOERROR + \ + libxml2.HTML_PARSE_NOWARNING + +def xmlDoc_from_html(response): + """Return libxml2 doc for HTMLs""" + try: + lxdoc = libxml2.htmlReadDoc(response.body.to_string('utf-8'), response.url, 'utf-8', html_parser_options) + except TypeError: # libxml2 doesn't parse text with null bytes + lxdoc = libxml2.htmlReadDoc(response.body.to_string('utf-8').replace("\x00", ""), response.url, 'utf-8', html_parser_options) + return lxdoc + +def xmlDoc_from_xml(response): + """Return libxml2 doc for XMLs""" + return libxml2.readDoc(response.body.to_string('utf-8'), response.url, 'utf-8', xml_parser_options) + diff --git a/scrapy/trunk/scrapy/xpath/document.py b/scrapy/trunk/scrapy/xpath/document.py new file mode 100644 index 000000000..f35ac8bc3 --- /dev/null +++ b/scrapy/trunk/scrapy/xpath/document.py @@ -0,0 +1,22 @@ +""" +This module contains a simple class (Libxml2Document) to wrap libxml2 documents +(xmlDoc) for proper garbage collection. +""" + +from scrapy.xpath.constructors import xmlDoc_from_html + +class Libxml2Document(object): + + def __init__(self, response, constructor=xmlDoc_from_html): + self.xmlDoc = constructor(response) + self.xpathContext = self.xmlDoc.xpathNewContext() + + def __del__(self): + if hasattr(self, 'xmlDoc'): + self.xmlDoc.freeDoc() + if hasattr(self, 'xpathContext'): + self.xpathContext.xpathFreeContext() + + def __str__(self): + return "<Libxml2Document %s>" % self.xmlDoc.name + diff --git a/scrapy/trunk/scrapy/xpath/extension.py b/scrapy/trunk/scrapy/xpath/extension.py new file mode 100644 index 000000000..0642e0cd6 --- /dev/null +++ b/scrapy/trunk/scrapy/xpath/extension.py @@ -0,0 +1,20 @@ +""" +The ResponseLibxml2 extension causes the Response objects to grow a new method +("getlibxml2doc") which returns a (cached) libxml2 document of itself. +""" + +from scrapy.http import Response +from scrapy.xpath.document import Libxml2Document +from scrapy.xpath.constructors import xmlDoc_from_html + +class ResponseLibxml2(object): + def __init__(self): + setattr(Response, 'getlibxml2doc', getlibxml2doc) + +def getlibxml2doc(response, constructor=xmlDoc_from_html): + attr = '_lx2doc_%s' % constructor.__name__ + if not hasattr(response, attr): + lx2doc = Libxml2Document(response, constructor=constructor) + setattr(response, attr, lx2doc) + return getattr(response, attr) + diff --git a/scrapy/trunk/scrapy/xpath/iterator.py b/scrapy/trunk/scrapy/xpath/iterator.py new file mode 100644 index 000000000..c216cad22 --- /dev/null +++ b/scrapy/trunk/scrapy/xpath/iterator.py @@ -0,0 +1,116 @@ +import re +from xml.sax.saxutils import escape +from cStringIO import StringIO + +import libxml2 + +from scrapy.xpath.constructors import xml_parser_options, xmlDoc_from_xml +from scrapy.xpath.selector import XPathSelector + +class XMLNodeIterator(object): + """XMLNodeIterator provides a way to iterate over all nodes of the same + name (passed in the constructor) in a XML Response without parsing the + entire response in memory. The iterator returns XPathSelector objects. + + Usage example: + + for x in XMLNodeIterator(response, "product"): + i = ScrapedItem() + i.assign("id", x.x("@id")) + i.assign("name", x.x("./name/text()") + """ + + def __init__(self, response, node, chunk_size=2048): + self.response = response + self.node = node + self.chunk_size = 2048 + + def __iter__(self): + sax_parser = XMLNodeSAXParser(self.node, self.response) + contents = self.response.body.to_string() + ctxt = libxml2.createPushParser(sax_parser, '', 0, None) + ctxt.ctxtUseOptions(xml_parser_options) + for i in xrange(0, len(contents), self.chunk_size): + chunk = contents[i:i + self.chunk_size] + ctxt.parseChunk(chunk, len(chunk), 0) + while sax_parser.selectors: + yield sax_parser.selectors.pop(0) + ctxt.parseChunk('', 0, 1) + +class XMLNodeSAXParser(): + + xmldeclr_re = re.compile(r'<\?xml.*?\?>') + + def __init__(self, requested_nodename, response): + self.requested_nodename = requested_nodename + self.inside_requested_node = False + self.buffer = StringIO() + self.xml_declaration = self._extract_xmldecl(response.body.to_string()[0:4096]) + self.selectors = [] + + def startElement(self, name, attributes): + if name == self.requested_nodename: + self.inside_requested_node = True + self.buffer.close() + self.buffer = StringIO() + attributes = attributes or {} + attribute_strings = ["%s='%s'" % tuple(ka) for ka in attributes.items()] + self.buffer.write('<' + ' '.join([name] + attribute_strings) + '>') + + def endElement(self, name): + self.buffer.write('</%s>' % name) + + if name == self.requested_nodename: + self.inside_requested_node = False + string = ''.join([self.xml_declaration, self.buffer.getvalue()]) + selector = XPathSelector(text=string, constructor=xmlDoc_from_xml).x('/' + self.requested_nodename)[0] + self.selectors.append(selector) + + def characters(self, data): + if self.inside_requested_node: + self.buffer.write(escape(data)) + + def cdataBlock(self, data): + #self.characters('<![CDATA[' + data + ']]>') + if self.inside_requested_node: + self.buffer.write('<![CDATA[' + data + ']]>') + + def _extract_xmldecl(self, string): + m = self.xmldeclr_re.search(string) + return m.group() if m else '' + + +# TESTING # +from xml.parsers.expat import ParserCreate + +class expat_XMLNodeIterator(): + def __init__(self, response, req_nodename, chunk_size=2048): + self._response = response + self._req_nodename = req_nodename + self._chunk_size = chunk_size + + self._byte_offset_buffer = [] + + self._parser = ParserCreate() + self._parser.StartElementHandler = self._StartElementHandler + self._parser.EndElementHandler = self._EndElementHandler + + def _StartElementHandler(self, name, attrs): + if name == self._req_nodename and not self._inside_req_node: + self._start_pos = self._parser.CurrentByteIndex + self._inside_req_node = True + + def _EndElementHandler(self, name): + if name == self._req_nodename and self._inside_req_node: + self._byte_offset_buffer.append((self._start_pos, self._parser.CurrentByteIndex)) + self._inside_req_node = False + + def __iter__(self): + response_body = self._response.body.to_string() + self._inside_req_node = False + for i in xrange(0, len(response_body), self._chunk_size): + self._parser.Parse(response_body[i:i + self._chunk_size]) + while self._byte_offset_buffer: + start, end = self._byte_offset_buffer.pop(0) + yield response_body[start:end] + self._parser.Parse('', 1) diff --git a/scrapy/trunk/scrapy/xpath/selector.py b/scrapy/trunk/scrapy/xpath/selector.py new file mode 100644 index 000000000..85d0d8762 --- /dev/null +++ b/scrapy/trunk/scrapy/xpath/selector.py @@ -0,0 +1,87 @@ +import libxml2 + +from scrapy.http import Response +from scrapy.xpath.extension import Libxml2Document +from scrapy.xpath.constructors import xmlDoc_from_html +from scrapy.utils.python import flatten +from scrapy.utils.misc import extract_regex + +class XPathSelector(object): + """Provides an easy way for selecting document parts using XPaths and + regexs, it also supports nested queries. + + Usage example (untested code): + + x = XPathSelector(response) + i = ScrapedItem() + i.assign("name", x.x("//h2/text()")) + i.assign("features", x.x("//div[@class='features']).x("./span/text()") + """ + + + def __init__(self, response=None, text=None, node=None, parent=None, expr=None, constructor=xmlDoc_from_html): + if parent: + self.doc = parent.doc + self.xmlNode = node + elif response: + try: + self.doc = response.getlibxml2doc(constructor=constructor) # try with cached version first + except AttributeError: + self.doc = Libxml2Document(response, constructor=constructor) + self.xmlNode = self.doc.xmlDoc + elif text: + response = Response(domain=None, url=None, body=str(text)) + self.doc = Libxml2Document(response, constructor=constructor) + self.xmlNode = self.doc.xmlDoc + self.expr = expr + + def x(self, xpath): + if hasattr(self.xmlNode, 'xpathEval'): + self.doc.xpathContext.setContextNode(self.xmlNode) + xpath_result = self.doc.xpathContext.xpathEval(xpath) + if hasattr(xpath_result, '__iter__'): + return XPathSelectorList([XPathSelector(node=node, parent=self, expr=xpath) for node in xpath_result]) + else: + return XPathSelectorList([XPathSelector(node=xpath_result, parent=self, expr=xpath)]) + else: + return XPathSelectorList([]) + + def re(self, regex): + return extract_regex(regex, self.extract(), 'utf-8') + + def extract(self, **kwargs): + if isinstance(self.xmlNode, basestring): + text = unicode(self.xmlNode, 'utf-8', errors='ignore') + elif hasattr(self.xmlNode, 'xpathEval'): + if isinstance(self.xmlNode, libxml2.xmlAttr): + text = unicode(self.xmlNode.content, errors='ignore') + else: + data = self.xmlNode.serialize('utf-8') + text = unicode(data, 'utf-8', errors='ignore') if data else u'' + else: + try: + text = unicode(self.xmlNode, errors='ignore') + except TypeError: # catched when self.xmlNode is a float - see tests + text = unicode(self.xmlNode) + return text + + def register_namespace(self, prefix, uri): + self.doc.xpathContext.xpathRegisterNs(prefix, uri) + + def __str__(self): + return "<XPathSelector (%s) xpath=%s>" % (getattr(self.xmlNode, 'name'), self.expr) + + __repr__ = __str__ + + +class XPathSelectorList(list): + + def extract(self, **kwargs): + return [x.extract(**kwargs) if isinstance(x, XPathSelector) else x for x in self] + + def x(self, xpath): + return XPathSelectorList(flatten([x.x(xpath) for x in self])) + + def re(self, regex): + return flatten([x.re(regex) for x in self]) + diff --git a/scrapy/trunk/scrapy/xpath/types.py b/scrapy/trunk/scrapy/xpath/types.py new file mode 100644 index 000000000..56d2efe96 --- /dev/null +++ b/scrapy/trunk/scrapy/xpath/types.py @@ -0,0 +1,11 @@ +class XPath(object): + """A XPath expression""" + + def __init__(self, xpath_expr): + self.expr = xpath_expr + + def __repr__(self): + return "XPath(%s)" % repr(self.expr) + + def __str__(self): + return self.expr
    123