diff --git a/direct/src/plugin/Sources.pp b/direct/src/plugin/Sources.pp index 9ce625865a..2f6962d992 100644 --- a/direct/src/plugin/Sources.pp +++ b/direct/src/plugin/Sources.pp @@ -1,9 +1,9 @@ // This directory is still experimental. Define HAVE_P3D_PLUGIN in // your Config.pp to build it. -#define BUILD_DIRECTORY $[and $[HAVE_P3D_PLUGIN],$[HAVE_PYTHON],$[HAVE_TINYXML]] +#define BUILD_DIRECTORY $[and $[HAVE_P3D_PLUGIN],$[HAVE_PYTHON],$[HAVE_TINYXML],$[HAVE_OPENSSL],$[HAVE_ZLIB]] #begin lib_target - #define USE_PACKAGES tinyxml + #define USE_PACKAGES tinyxml openssl zlib #define TARGET p3d_plugin #define COMBINED_SOURCES \ @@ -18,6 +18,7 @@ p3dFileDownload.h p3dFileDownload.I \ p3dInstance.h p3dInstance.I \ p3dInstanceManager.h p3dInstanceManager.I \ + p3dPackage.h p3dPackage.I \ p3dSession.h p3dSession.I #define INCLUDED_SOURCES \ @@ -26,6 +27,7 @@ p3dFileDownload.cxx \ p3dInstance.cxx \ p3dInstanceManager.cxx \ + p3dPackage.cxx \ p3dSession.cxx #define INSTALL_HEADERS \ diff --git a/direct/src/plugin/make_package.py b/direct/src/plugin/make_package.py new file mode 100755 index 0000000000..2ead637b47 --- /dev/null +++ b/direct/src/plugin/make_package.py @@ -0,0 +1,190 @@ +#! /bin/env python + +""" +This command is used to build a downloadable package for the p3d +plugin to retrieve and install. It examines the files in the current +directory, assumes they are all intended to be part of the package, +and constructs the necessary package xml file and archive file for +hosting on the web serevr. + +make_package.py [opts] + +Options: + + -d stage_dir + + Specify the staging directory. This is a temporary directory on + the local machine that will be filled with the contents of the + package directory for the web server. + + -p package_name + -v package_version + + Specify the name and version of the package to build. + +""" + +import sys +import getopt +import os +import tarfile +import gzip +import stat +import md5 + +import direct +from pandac.PandaModules import * + +class ArgumentError(AttributeError): + pass + +class FileSpec: + def __init__(self, filename, pathname): + self.filename = filename + self.pathname = pathname + self.size = 0 + self.timestamp = 0 + self.hash = None + + s = os.stat(self.pathname) + self.size = s[stat.ST_SIZE] + self.timestamp = s[stat.ST_MTIME] + + m = md5.new() + f = open(self.pathname, 'rb') + data = f.read(4096) + while data: + m.update(data) + data = f.read(4096) + f.close() + + self.hash = m.hexdigest() + + def get_params(self): + return 'filename="%s" size=%s timestamp=%s hash="%s"' % ( + self.filename, self.size, self.timestamp, self.hash) + +class PackageMaker: + def __init__(self): + self.startDir = None + self.stageDir = None + self.packageName = None + self.packageVersion = None + + + def build(self): + if not self.startDir: + raise ArgumentError, "Start directory not specified." + if not self.stageDir: + raise ArgumentError, "Stage directory not specified." + if not self.packageName or not self.packageVersion: + raise ArgumentError, "Package name and version not specified." + + self.packageFullname = '%s_%s' % ( + self.packageName, self.packageVersion) + + self.cleanDir(self.stageDir) + + uncompressedArchiveBasename = '%s.tar' % (self.packageFullname) + uncompressedArchivePathname = os.path.join(self.stageDir, uncompressedArchiveBasename) + self.archive = tarfile.open(uncompressedArchivePathname, 'w') + + self.components = [] + + self.addComponents() + self.archive.close() + + uncompressedArchive = FileSpec(uncompressedArchiveBasename, uncompressedArchivePathname) + + compressedArchiveBasename = '%s.tgz' % (self.packageFullname) + compressedArchivePathname = os.path.join(self.stageDir, compressedArchiveBasename) + + print "\ncompressing" + f = open(uncompressedArchivePathname, 'rb') + gz = gzip.open(compressedArchivePathname, 'w', 9) + data = f.read(4096) + while data: + gz.write(data) + data = f.read(4096) + gz.close() + f.close() + + compressedArchive = FileSpec(compressedArchiveBasename, compressedArchivePathname) + + os.unlink(uncompressedArchivePathname) + + descFileBasename = '%s.xml' % (self.packageFullname) + descFilePathname = os.path.join(self.stageDir, descFileBasename) + + f = open(descFilePathname, 'w') + print >> f, '' + print >> f, '' + print >> f, '' % (self.packageName, self.packageVersion) + print >> f, ' ' % (uncompressedArchive.get_params()) + print >> f, ' ' % (compressedArchive.get_params()) + for file in self.components: + print >> f, ' ' % (file.get_params()) + print >> f, '' + f.close() + + + def cleanDir(self, dirname): + """ Remove all the files in the named directory. Does not + operate recursively. """ + + for filename in os.listdir(dirname): + pathname = os.path.join(dirname, filename) + try: + os.unlink(pathname) + except OSError: + pass + + def addComponents(self): + """ Walks through all the files in the start directory and + adds them to the archive. Recursively visits + sub-directories. """ + + startDir = self.startDir + if startDir.endswith(os.altsep) or startDir.endswith(os.sep): + startDir = startDir[:-1] + prefix = startDir + os.sep + for dirpath, dirnames, filenames in os.walk(startDir): + if dirpath == startDir: + localpath = '' + else: + assert dirpath.startswith(prefix) + localpath = dirpath[len(prefix):] + + for basename in filenames: + file = FileSpec(os.path.join(localpath, basename), + os.path.join(startDir, basename)) + print file.filename + self.components.append(file) + self.archive.add(file.pathname, file.filename, recursive = False) + +def makePackage(args): + opts, args = getopt.getopt(args, 'd:p:v:h') + + pm = PackageMaker() + pm.startDir = '.' + for option, value in opts: + if option == '-d': + pm.stageDir = Filename.fromOsSpecific(value).toOsSpecific() + elif option == '-p': + pm.packageName = value + elif option == '-v': + pm.packageVersion = value + + elif option == '-h': + print __doc__ + sys.exit(1) + + pm.build() + + +if __name__ == '__main__': + try: + makePackage(sys.argv[1:]) + except ArgumentError, e: + print e.args[0] + sys.exit(1) diff --git a/direct/src/plugin/p3dDownload.cxx b/direct/src/plugin/p3dDownload.cxx index e256063ab8..b5053533d5 100755 --- a/direct/src/plugin/p3dDownload.cxx +++ b/direct/src/plugin/p3dDownload.cxx @@ -25,7 +25,8 @@ P3DDownload() { _http_status_code = 0; _total_data = 0; _total_expected_data = 0; - + + _canceled = false; _download_id = 0; } @@ -48,6 +49,21 @@ set_url(const string &url) { _url = url; } +//////////////////////////////////////////////////////////////////// +// Function: P3DDownload::cancel +// Access: Public +// Description: Cancels a running download. download_finished() will +// not be called, but the P3DDownload object itself will +// eventually be deleted by its owning P3DInstance. +//////////////////////////////////////////////////////////////////// +void P3DDownload:: +cancel() { + _canceled = true; + if (_status == P3D_RC_in_progress) { + _status = P3D_RC_generic_error; + } +} + //////////////////////////////////////////////////////////////////// // Function: P3DDownload::feed_url_stream // Access: Public @@ -60,12 +76,16 @@ feed_url_stream(P3D_result_code result_code, size_t total_expected_data, const unsigned char *this_data, size_t this_data_size) { + if (_canceled) { + return false; + } + _status = result_code; _http_status_code = http_status_code; if (this_data_size != 0) { if (!receive_data(this_data, this_data_size)) { - // Failure writing to disk. + // Failure writing to disk or some such. _status = P3D_RC_generic_error; download_finished(false); return false; diff --git a/direct/src/plugin/p3dDownload.h b/direct/src/plugin/p3dDownload.h index 662cdf2fd9..aa44a8f2ce 100755 --- a/direct/src/plugin/p3dDownload.h +++ b/direct/src/plugin/p3dDownload.h @@ -37,6 +37,8 @@ public: inline bool get_download_finished() const; inline bool get_download_success() const; + void cancel(); + public: // These are intended to be called only by P3DInstance. inline void set_download_id(int download_id); @@ -62,6 +64,7 @@ protected: size_t _total_expected_data; private: + bool _canceled; int _download_id; string _url; }; diff --git a/direct/src/plugin/p3dInstanceManager.I b/direct/src/plugin/p3dInstanceManager.I index 783c7b98f1..6f9d307dbc 100644 --- a/direct/src/plugin/p3dInstanceManager.I +++ b/direct/src/plugin/p3dInstanceManager.I @@ -13,13 +13,51 @@ //////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +// Function: P3DInstanceManager::get_root_dir +// Access: Public +// Description: Returns the root directory into which all the P3D +// runtime files are downloaded and installed. This +// must be a writable directory or nothing will work. +//////////////////////////////////////////////////////////////////// +inline const string &P3DInstanceManager:: +get_root_dir() const { + return _root_dir; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DInstanceManager::get_download_url +// Access: Public +// Description: Returns the URL of the download server. All +// downloadable files will be retrieved from various +// subdirectories of this URL root. +//////////////////////////////////////////////////////////////////// +inline const string &P3DInstanceManager:: +get_download_url() const { + return _download_url; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DInstanceManager::get_command_instance +// Access: Public +// Description: Returns a global P3DInstance object that exists for +// the lifetime of the instance manager. This instance +// object has no reality onscreen, but can be used as a +// download host for downloading files that aren't +// specifically needed by any user-created instance. +//////////////////////////////////////////////////////////////////// +inline P3DInstance *P3DInstanceManager:: +get_command_instance() const { + return _command_instance; +} + //////////////////////////////////////////////////////////////////// // Function: P3DInstanceManager::get_num_instances // Access: Public // Description: Returns the number of instances currently running // within the world. //////////////////////////////////////////////////////////////////// -INLINE int P3DInstanceManager:: +inline int P3DInstanceManager:: get_num_instances() const { return _instances.size(); } diff --git a/direct/src/plugin/p3dInstanceManager.cxx b/direct/src/plugin/p3dInstanceManager.cxx index 1c9e5d7a9d..2a74b263c9 100644 --- a/direct/src/plugin/p3dInstanceManager.cxx +++ b/direct/src/plugin/p3dInstanceManager.cxx @@ -15,6 +15,7 @@ #include "p3dInstanceManager.h" #include "p3dInstance.h" #include "p3dSession.h" +#include "p3dPackage.h" P3DInstanceManager *P3DInstanceManager::_global_ptr; @@ -35,6 +36,8 @@ P3DInstanceManager() { INIT_LOCK(_request_ready_lock); pthread_cond_init(&_request_ready_cvar, NULL); #endif + + _command_instance = NULL; } //////////////////////////////////////////////////////////////////// @@ -44,7 +47,14 @@ P3DInstanceManager() { //////////////////////////////////////////////////////////////////// P3DInstanceManager:: ~P3DInstanceManager() { - // Actually, the destructor is never called. + // Actually, this destructor is never called, since this is a global + // object that never gets deleted. + + assert(_instances.empty()); + assert(_sessions.empty()); + + delete _command_instance; + #ifdef _WIN32 CloseHandle(_request_ready); #else @@ -63,6 +73,8 @@ P3DInstanceManager:: //////////////////////////////////////////////////////////////////// bool P3DInstanceManager:: initialize() { + _root_dir = "c:/cygwin/home/drose/p3ddir"; + _download_url = "http://10.196.143.118/~drose/"; return true; } @@ -109,6 +121,8 @@ create_instance(P3D_request_ready_func *func, //////////////////////////////////////////////////////////////////// void P3DInstanceManager:: finish_instance(P3DInstance *inst) { + assert(inst != _command_instance); + Instances::iterator ii; ii = _instances.find(inst); assert(ii != _instances.end()); @@ -145,6 +159,10 @@ check_request() { } } + if (_command_instance->has_request()) { + return _command_instance; + } + return NULL; } @@ -164,10 +182,10 @@ wait_request() { int seq = _request_seq; while (true) { - if (_instances.empty()) { + if (check_request() != (P3DInstance *)NULL) { return; } - if (check_request() != (P3DInstance *)NULL) { + if (_instances.empty()) { return; } @@ -187,6 +205,27 @@ wait_request() { } } +//////////////////////////////////////////////////////////////////// +// Function: P3DInstanceManager::get_package +// Access: Public +// Description: Returns a (possibly shared) pointer to the indicated +// package. +//////////////////////////////////////////////////////////////////// +P3DPackage *P3DInstanceManager:: +get_package(const string &package_name, const string &package_version) { + string key = package_name + "_" + package_version; + Packages::iterator pi = _packages.find(key); + if (pi != _packages.end()) { + return (*pi).second; + } + + P3DPackage *package = new P3DPackage(package_name, package_version); + bool inserted = _packages.insert(Packages::value_type(key, package)).second; + assert(inserted); + + return package; +} + //////////////////////////////////////////////////////////////////// // Function: P3DInstanceManager::get_unique_session_index // Access: Public @@ -229,6 +268,22 @@ P3DInstanceManager *P3DInstanceManager:: get_global_ptr() { if (_global_ptr == NULL) { _global_ptr = new P3DInstanceManager; + _global_ptr->create_command_instance(); } return _global_ptr; } + +//////////////////////////////////////////////////////////////////// +// Function: P3DInstanceManager::create_command_instance; +// Access: Private +// Description: Create a command instance. This is used to handle +// requests that have nothing to do with any particular +// host-created instance. +//////////////////////////////////////////////////////////////////// +void P3DInstanceManager:: +create_command_instance() { + P3D_window_handle dummy_handle; + _command_instance = + new P3DInstance(NULL, "", P3D_WT_hidden, 0, 0, 0, 0, + dummy_handle, NULL, 0); +} diff --git a/direct/src/plugin/p3dInstanceManager.h b/direct/src/plugin/p3dInstanceManager.h index 6c73c63a8a..7d1deb41cf 100644 --- a/direct/src/plugin/p3dInstanceManager.h +++ b/direct/src/plugin/p3dInstanceManager.h @@ -22,6 +22,7 @@ class P3DInstance; class P3DSession; +class P3DPackage; //////////////////////////////////////////////////////////////////// // Class : P3DInstanceManager @@ -36,6 +37,9 @@ private: public: bool initialize(); + inline const string &get_root_dir() const; + inline const string &get_download_url() const; + P3DInstance * create_instance(P3D_request_ready_func *func, const string &p3d_filename, @@ -51,7 +55,11 @@ public: P3DInstance *check_request(); void wait_request(); - INLINE int get_num_instances() const; + P3DPackage *get_package(const string &package_name, + const string &package_version); + + inline P3DInstance *get_command_instance() const; + inline int get_num_instances() const; int get_unique_session_index(); void signal_request_ready(); @@ -59,7 +67,13 @@ public: static P3DInstanceManager *get_global_ptr(); private: - string _p3d_root_directory; + void create_command_instance(); + +private: + string _root_dir; + string _download_url; + + P3DInstance *_command_instance; typedef set Instances; Instances _instances; @@ -67,6 +81,9 @@ private: typedef map Sessions; Sessions _sessions; + typedef map Packages; + Packages _packages; + int _unique_session_index; // Implements a condition-var like behavior. diff --git a/direct/src/plugin/p3dPackage.I b/direct/src/plugin/p3dPackage.I new file mode 100755 index 0000000000..821d5a5d1e --- /dev/null +++ b/direct/src/plugin/p3dPackage.I @@ -0,0 +1,71 @@ +// Filename: p3dPackage.I +// Created by: drose (12Jun09) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::get_ready +// Access: Public +// Description: Returns true if the package has been downloaded and +// verified and is ready to be used, false if it has +// not. +//////////////////////////////////////////////////////////////////// +inline bool P3DPackage:: +get_ready() const { + return _ready; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::get_failed +// Access: Public +// Description: Returns true if the package cannot be made ready, for +// instance because the download server is down. +//////////////////////////////////////////////////////////////////// +inline bool P3DPackage:: +get_failed() const { + return _failed; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::decode_hexdigit +// Access: Private +// Description: Returns the integer value corresponding to the +// indicated hex digit. Returns -1 if it is not a hex +// digit. +//////////////////////////////////////////////////////////////////// +inline int P3DPackage:: +decode_hexdigit(char c) { + if (isdigit(c)) { + return c - '0'; + } + c = tolower(c); + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + + return -1; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::encode_hexdigit +// Access: Private +// Description: Returns the hex digit corresponding to the +// indicated integer value. +//////////////////////////////////////////////////////////////////// +inline char P3DPackage:: +encode_hexdigit(int c) { + if (c >= 10) { + return c - 10 + 'a'; + } + return c + '0'; +} diff --git a/direct/src/plugin/p3dPackage.cxx b/direct/src/plugin/p3dPackage.cxx new file mode 100755 index 0000000000..0044cc1fdd --- /dev/null +++ b/direct/src/plugin/p3dPackage.cxx @@ -0,0 +1,695 @@ +// Filename: p3dPackage.cxx +// Created by: drose (12Jun09) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#include "p3dPackage.h" +#include "p3dInstanceManager.h" + +#include "openssl/md5.h" +#include "zlib.h" + +#include +#include + +#include +#include +#include + +#ifdef _WIN32 +#include +#define stat _stat +#define utime _utime +#define utimbuf _utimbuf +#endif + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +P3DPackage:: +P3DPackage(const string &package_name, const string &package_version) : + _package_name(package_name), + _package_version(package_version) +{ + _package_fullname = _package_name + "_" + _package_version; + _ready = false; + _failed = false; + _active_download = NULL; + _partial_download = false; + + P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr(); + + // Ensure the package directory exists; create it if it does not. + _package_dir = inst_mgr->get_root_dir() + string("/") + _package_name; +#ifdef _WIN32 + _mkdir(_package_dir.c_str()); +#else + mkdir(_package_dir.c_str(), 0777); +#endif + + _package_dir += string("/") + _package_version; +#ifdef _WIN32 + _mkdir(_package_dir.c_str()); +#else + mkdir(_package_dir.c_str(), 0777); +#endif + + _desc_file_basename = _package_fullname + ".xml"; + _desc_file_pathname = _package_dir + "/" + _desc_file_basename; + + // TODO: we should check the desc file for updates with the server. + // Perhaps this should be done in a parent class. + + // Load the desc file, if it exists. + TiXmlDocument doc(_desc_file_pathname.c_str()); + if (!doc.LoadFile()) { + download_desc_file(); + } else { + got_desc_file(&doc, false); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::Destructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +P3DPackage:: +~P3DPackage() { + // Tell any pending callbacks that we're no good any more. + report_done(false); + + // Cancel any pending download. + if (_active_download != NULL) { + _active_download->cancel(); + delete _active_download; + _active_download = NULL; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::set_callback +// Access: Public +// Description: Registers a callback on the package. The callback +// object will be notified when the package is ready for +// use (or when it has failed to download properly). +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +set_callback(Callback *callback) { + if (_ready || _failed) { + // Actually, we're already done. Signal the callback immediately. + callback->package_ready(this, _ready); + } else { + // Bootstrap still in progress. Save the callback. + _callbacks.push_back(callback); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::cancel_callback +// Access: Public +// Description: Unregisters a particular callback object. This +// object will no longer be notified when the package is +// ready. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +cancel_callback(Callback *callback) { + Callbacks::iterator ci; + ci = find(_callbacks.begin(), _callbacks.end(), callback); + if (ci != _callbacks.end()) { + _callbacks.erase(ci); + } else { + cerr << "Canceling unknown callback on " << _package_fullname << "\n"; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::download_desc_file +// Access: Private +// Description: Starts downloading the desc file for the package. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +download_desc_file() { + P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr(); + string url = inst_mgr->get_download_url(); + url += _package_name + "/" + _package_version + "/" + _desc_file_basename; + + start_download(DT_desc_file, url, _desc_file_pathname, false); +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::desc_file_download_finished +// Access: Private +// Description: Called when the desc file has been fully downloaded. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +desc_file_download_finished(bool success) { + if (!success) { + report_done(false); + return; + } + + TiXmlDocument doc(_desc_file_pathname.c_str()); + if (!doc.LoadFile()) { + cerr << "Couldn't read " << _desc_file_pathname << "\n"; + report_done(false); + return; + } + + got_desc_file(&doc, true); +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::got_desc_file +// Access: Private +// Description: Reads the desc file and begins verifying the files. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +got_desc_file(TiXmlDocument *doc, bool freshly_downloaded) { + cerr << "got desc file\n"; + + TiXmlElement *xpackage = doc->FirstChildElement("package"); + TiXmlElement *uncompressed_archive = NULL; + TiXmlElement *compressed_archive = NULL; + + if (xpackage != NULL) { + uncompressed_archive = xpackage->FirstChildElement("uncompressed_archive"); + compressed_archive = xpackage->FirstChildElement("compressed_archive"); + } + + if (uncompressed_archive == NULL || compressed_archive == NULL) { + // The desc file didn't include the archive file itself, weird. + if (!freshly_downloaded) { + download_desc_file(); + return; + } + report_done(false); + return; + } + + _uncompressed_archive.load_xml(uncompressed_archive); + _compressed_archive.load_xml(compressed_archive); + + // Now get all the components. + _components.clear(); + TiXmlElement *component = xpackage->FirstChildElement("component"); + while (component != NULL) { + FileSpec file; + file.load_xml(component); + _components.push_back(file); + component = component->NextSiblingElement("component"); + } + + cerr << "got " << _components.size() << " components\n"; + + // Verify all of the components. + bool all_components_ok = true; + Components::iterator ci; + for (ci = _components.begin(); ci != _components.end(); ++ci) { + if (!(*ci).quick_verify(_package_dir)) { + all_components_ok = false; + break; + } + } + + if (all_components_ok) { + // Great, we're ready to begin. + report_done(true); + + } else if (_uncompressed_archive.quick_verify(_package_dir)) { + // We need to re-extract the archive. + extract_archive(); + + } else if (_compressed_archive.quick_verify(_package_dir)) { + // We need to uncompress the archive. + uncompress_archive(); + + } else { + // Shoot, we need to download the archive. + download_compressed_archive(true); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::download_compressed_archive +// Access: Private +// Description: Starts downloading the archive file for the package. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +download_compressed_archive(bool allow_partial) { + P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr(); + string url = inst_mgr->get_download_url(); + url += _package_name + "/" + _package_version + "/" + _compressed_archive._filename; + + string target_pathname = _package_dir + "/" + _compressed_archive._filename; + + start_download(DT_compressed_archive, url, target_pathname, allow_partial); +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::compressed_archive_download_finished +// Access: Private +// Description: Called when the desc file has been fully downloaded. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +compressed_archive_download_finished(bool success) { + if (!success) { + report_done(false); + return; + } + + if (_compressed_archive.full_verify(_package_dir)) { + // Go on to uncompress the archive. + uncompress_archive(); + return; + } + + // Oof, didn't download it correctly. + if (_partial_download) { + // Go back and get the whole file this time. + download_compressed_archive(false); + } + + cerr << _compressed_archive._filename + << " failed hash check after download\n"; + report_done(false); +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::uncompress_archive +// Access: Private +// Description: Uncompresses the archive file. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +uncompress_archive() { + cerr << "uncompressing " << _compressed_archive._filename << "\n"; + + string source_pathname = _package_dir + "/" + _compressed_archive._filename; + string target_pathname = _package_dir + "/" + _uncompressed_archive._filename; + + gzFile source = gzopen(source_pathname.c_str(), "rb"); + if (source == NULL) { + cerr << "Couldn't open " << source_pathname << "\n"; + report_done(false); + return; + } + + ofstream target(target_pathname.c_str(), ios::out | ios::binary); + if (!target) { + cerr << "Couldn't write to " << target_pathname << "\n"; + report_done(false); + } + + static const int buffer_size = 1024; + char buffer[buffer_size]; + + int count = gzread(source, buffer, buffer_size); + while (count > 0) { + target.write(buffer, count); + count = gzread(source, buffer, buffer_size); + } + + if (count < 0) { + cerr << "gzip error decompressing " << source_pathname << "\n"; + int errnum; + cerr << gzerror(source, &errnum) << "\n"; + gzclose(source); + report_done(false); + return; + } + + gzclose(source); + + if (!target) { + cerr << "Couldn't write entire file to " << target_pathname << "\n"; + report_done(false); + return; + } + + target.close(); + + if (!_uncompressed_archive.full_verify(_package_dir)) { + cerr << "after uncompressing " << target_pathname << ", failed hash check\n"; + report_done(false); + return; + } + + unlink(source_pathname.c_str()); + + // All done uncompressing. + extract_archive(); +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::extract_archive +// Access: Private +// Description: Extracts the components from the archive file. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +extract_archive() { + cerr << "extracting " << _uncompressed_archive._filename << "\n"; + +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::report_done +// Access: Private +// Description: Transitions the package to "ready" or "failure" +// state, and reports this change to all the listening +// callbacks. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +report_done(bool success) { + if (success) { + _ready = true; + _failed = false; + } else { + _ready = false; + _failed = true; + } + + Callbacks orig_callbacks; + orig_callbacks.swap(_callbacks); + Callbacks::iterator ci; + for (ci = orig_callbacks.begin(); ci != orig_callbacks.end(); ++ci) { + (*ci)->package_ready(this, _ready); + } + + // We shouldn't have added any more callbacks during the above loop. + assert(_callbacks.empty()); +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::start_download +// Access: Private +// Description: Initiates a download of the indicated file. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +start_download(P3DPackage::DownloadType dtype, const string &url, + const string &pathname, bool allow_partial) { + // Only one download should be active at a time + assert(_active_download == NULL); + + Download *download = new Download(this, dtype); + download->set_url(url); + download->set_filename(pathname); + + // TODO: implement partial file re-download. + allow_partial = false; + + if (!allow_partial) { + unlink(pathname.c_str()); + } + + _active_download = download; + _partial_download = false; + P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr(); + inst_mgr->get_command_instance()->start_download(download); +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::decode_hex +// Access: Private, Static +// Description: Decodes the hex string in source into the character +// array in dest. dest must have has least size bytes; +// source must have size * 2 bytes. +// +// Returns true on success, false if there was a non-hex +// digit in the string. +//////////////////////////////////////////////////////////////////// +bool P3DPackage:: +decode_hex(unsigned char *dest, const char *source, size_t size) { + for (size_t i = 0; i < size; ++i) { + int high = decode_hexdigit(source[i * 2]); + int low = decode_hexdigit(source[i * 2 + 1]); + if (high < 0 || low < 0) { + return false; + } + dest[i] = (high << 4) | low; + } + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::encode_hex +// Access: Private, Static +// Description: Encodes a character array into a hex string for +// output. dest must have at least size * 2 bytes; +// source must have size bytes. The result is not +// null-terminated. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +encode_hex(char *dest, const unsigned char *source, size_t size) { + for (size_t i = 0; i < size; ++i) { + int high = (source[i] >> 4) & 0xf; + int low = source[i] & 0xf; + dest[2 * i] = encode_hexdigit(high); + dest[2 * i + 1] = encode_hexdigit(low); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::stream_hex +// Access: Private, Static +// Description: Writes the indicated buffer as a string of hex +// characters to the given ostream. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +stream_hex(ostream &out, const unsigned char *source, size_t size) { + for (size_t i = 0; i < size; ++i) { + int high = (source[i] >> 4) & 0xf; + int low = source[i] & 0xf; + out.put(encode_hexdigit(high)); + out.put(encode_hexdigit(low)); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::Download::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +P3DPackage::Download:: +Download(P3DPackage *package, DownloadType dtype) : + _package(package), + _dtype(dtype) +{ +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::Download::download_finished +// Access: Protected, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +void P3DPackage::Download:: +download_finished(bool success) { + P3DFileDownload::download_finished(success); + assert(_package->_active_download == this); + _package->_active_download = NULL; + + switch (_dtype) { + case DT_desc_file: + _package->desc_file_download_finished(success); + break; + + case DT_compressed_archive: + _package->compressed_archive_download_finished(success); + break; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::FileSpec::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +P3DPackage::FileSpec:: +FileSpec() { + _size = 0; + _timestamp = 0; + memset(_hash, 0, sizeof(_hash)); + _got_hash = false; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::FileSpec::load_xml +// Access: Public +// Description: Reads the data from the indicated XML file. +//////////////////////////////////////////////////////////////////// +void P3DPackage::FileSpec:: +load_xml(TiXmlElement *element) { + const char *filename = element->Attribute("filename"); + if (filename != NULL) { + _filename = filename; + } + + const char *size = element->Attribute("size"); + if (size != NULL) { + char *endptr; + _size = strtoul(size, &endptr, 10); + } + + const char *timestamp = element->Attribute("timestamp"); + if (timestamp != NULL) { + char *endptr; + _timestamp = strtoul(timestamp, &endptr, 10); + } + + _got_hash = false; + const char *hash = element->Attribute("hash"); + if (hash != NULL && strlen(hash) == (hash_size * 2)) { + // Decode the hex hash string. + _got_hash = decode_hex(_hash, hash, hash_size); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::FileSpec::quick_verify +// Access: Public +// Description: Performs a quick test to ensure the file has not been +// modified. This test is vulnerable to people +// maliciously attempting to fool the program (by +// setting datestamps etc.). +// +// Returns true if it is intact, false if it needs to be +// redownloaded. +//////////////////////////////////////////////////////////////////// +bool P3DPackage::FileSpec:: +quick_verify(const string &package_dir) const { + string pathname = package_dir + "/" + _filename; + struct stat st; + if (stat(pathname.c_str(), &st) != 0) { + cerr << "file not found: " << _filename << "\n"; + return false; + } + + if (st.st_size != _size) { + // If the size is wrong, the file fails. + cerr << "size wrong: " << _filename << "\n"; + return false; + } + + if (st.st_mtime == _timestamp) { + // If the size is right and the timestamp is right, the file passes. + return true; + } + + cerr << "modification time wrong: " << _filename << "\n"; + + // If the size is right but the timestamp is wrong, the file + // soft-fails. We follow this up with a hash check. + if (!check_hash(pathname)) { + // Hard fail, the hash is wrong. + cerr << "hash check wrong: " << _filename << "\n"; + return false; + } + + cerr << "hash check ok: " << _filename << "\n"; + + // The hash is OK after all. Change the file's timestamp back to + // what we expect it to be, so we can quick-verify it successfully + // next time. + utimbuf utb; + utb.actime = st.st_atime; + utb.modtime = _timestamp; + utime(pathname.c_str(), &utb); + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::FileSpec::quick_verify +// Access: Public +// Description: Performs a more thorough test to ensure the file has +// not been modified. This test is less vulnerable to +// malicious attacks, since it reads and verifies the +// entire file. +// +// Returns true if it is intact, false if it needs to be +// redownloaded. +//////////////////////////////////////////////////////////////////// +bool P3DPackage::FileSpec:: +full_verify(const string &package_dir) const { + string pathname = package_dir + "/" + _filename; + struct stat st; + if (stat(pathname.c_str(), &st) != 0) { + cerr << "file not found: " << _filename << "\n"; + return false; + } + + if (st.st_size != _size) { + // If the size is wrong, the file fails. + cerr << "size wrong: " << _filename << "\n"; + return false; + } + + // If the size is right but the timestamp is wrong, the file + // soft-fails. We follow this up with a hash check. + if (!check_hash(pathname)) { + // Hard fail, the hash is wrong. + cerr << "hash check wrong: " << _filename << "\n"; + return false; + } + + cerr << "hash check ok: " << _filename << "\n"; + + // The hash is OK. If the timestamp is wrong, change it back to + // what we expect it to be, so we can quick-verify it successfully + // next time. + + if (st.st_mtime != _timestamp) { + utimbuf utb; + utb.actime = st.st_atime; + utb.modtime = _timestamp; + utime(pathname.c_str(), &utb); + } + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::FileSpec::check_hash +// Access: Public +// Description: Returns true if the file has the expected md5 hash, +// false otherwise. +//////////////////////////////////////////////////////////////////// +bool P3DPackage::FileSpec:: +check_hash(const string &pathname) const { + ifstream stream(pathname.c_str(), ios::in | ios::binary); + if (!stream) { + cerr << "unable to read " << pathname << "\n"; + return false; + } + + unsigned char md[hash_size]; + + MD5_CTX ctx; + MD5_Init(&ctx); + + static const int buffer_size = 1024; + char buffer[buffer_size]; + + stream.read(buffer, buffer_size); + size_t count = stream.gcount(); + while (count != 0) { + MD5_Update(&ctx, buffer, count); + stream.read(buffer, buffer_size); + count = stream.gcount(); + } + + MD5_Final(md, &ctx); + + return (memcmp(md, _hash, hash_size) == 0); +} diff --git a/direct/src/plugin/p3dPackage.h b/direct/src/plugin/p3dPackage.h new file mode 100755 index 0000000000..32b0c4a5ae --- /dev/null +++ b/direct/src/plugin/p3dPackage.h @@ -0,0 +1,135 @@ +// Filename: p3dPackage.h +// Created by: drose (12Jun09) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#ifndef P3DPACKAGE_H +#define P3DPACKAGE_H + +#include "p3d_plugin_common.h" +#include "p3dFileDownload.h" + +//////////////////////////////////////////////////////////////////// +// Class : P3DPackage +// Description : This corresponds to a downloadable, patchable +// package, and all its constituent files. For +// instance, a particular version of the Panda3D +// runtime, which consists of a bunch of dll's +// downloaded in a single tar file, is a package. +// +// The plugin is responsible for managing these packages +// on disk, downloading new versions when needed, and +// removing stale versions to limit disk space waste. +//////////////////////////////////////////////////////////////////// +class P3DPackage { +public: + P3DPackage(const string &package_name, const string &package_version); + ~P3DPackage(); + + class Callback { + public: + virtual void package_ready(P3DPackage *package, bool success); + }; + + inline bool get_ready() const; + inline bool get_failed() const; + + void set_callback(Callback *callback); + void cancel_callback(Callback *callback); + +private: + enum DownloadType { + DT_desc_file, + DT_compressed_archive + }; + + class Download : public P3DFileDownload { + public: + Download(P3DPackage *package, DownloadType dtype); + + protected: + virtual void download_finished(bool success); + + private: + P3DPackage *_package; + DownloadType _dtype; + }; + + void download_desc_file(); + void desc_file_download_finished(bool success); + void got_desc_file(TiXmlDocument *doc, bool freshly_downloaded); + + void download_compressed_archive(bool allow_partial); + void compressed_archive_download_finished(bool success); + + void uncompress_archive(); + void extract_archive(); + + void report_done(bool success); + void start_download(DownloadType dtype, const string &url, + const string &pathname, bool allow_partial); + + static inline int decode_hexdigit(char c); + static inline char encode_hexdigit(int c); + + static bool decode_hex(unsigned char *dest, const char *source, size_t size); + static void encode_hex(char *dest, const unsigned char *source, size_t size); + static void stream_hex(ostream &out, const unsigned char *source, size_t size); + +private: + string _package_name; + string _package_version; + string _package_fullname; + string _package_dir; + + string _desc_file_basename; + string _desc_file_pathname; + + bool _ready; + bool _failed; + Download *_active_download; + bool _partial_download; + + typedef vector Callbacks; + Callbacks _callbacks; + + enum { hash_size = 16 }; + + class FileSpec { + public: + FileSpec(); + void load_xml(TiXmlElement *element); + + bool quick_verify(const string &package_dir) const; + bool full_verify(const string &package_dir) const; + + bool check_hash(const string &pathname) const; + + string _filename; + size_t _size; + time_t _timestamp; + unsigned char _hash[hash_size]; + bool _got_hash; + }; + + FileSpec _compressed_archive; + FileSpec _uncompressed_archive; + + typedef vector Components; + Components _components; + + friend class Download; +}; + +#include "p3dPackage.I" + +#endif diff --git a/direct/src/plugin/p3dSession.cxx b/direct/src/plugin/p3dSession.cxx index 4373aa3baa..f11d11441f 100644 --- a/direct/src/plugin/p3dSession.cxx +++ b/direct/src/plugin/p3dSession.cxx @@ -131,9 +131,9 @@ start_instance(P3DInstance *inst) { send_command(doc); - if (_python_state == PS_init) { - download_p3dpython(inst); - } + P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr(); + P3DPackage *panda = inst_mgr->get_package("panda3d", "dev"); + // start_p3dpython(); } //////////////////////////////////////////////////////////////////// @@ -209,7 +209,7 @@ download_p3dpython(P3DInstance *inst) { inst->start_download(download); */ - start_p3dpython(); + // start_p3dpython(); } //////////////////////////////////////////////////////////////////// diff --git a/direct/src/plugin/p3d_plugin_composite1.cxx b/direct/src/plugin/p3d_plugin_composite1.cxx index cb18b8413e..3ff57ea1e3 100644 --- a/direct/src/plugin/p3d_plugin_composite1.cxx +++ b/direct/src/plugin/p3d_plugin_composite1.cxx @@ -3,4 +3,5 @@ #include "p3dFileDownload.cxx" #include "p3dInstance.cxx" #include "p3dInstanceManager.cxx" +#include "p3dPackage.cxx" #include "p3dSession.cxx" diff --git a/direct/src/plugin/panda3d.cxx b/direct/src/plugin/panda3d.cxx index 34db46e135..2cba11fcce 100644 --- a/direct/src/plugin/panda3d.cxx +++ b/direct/src/plugin/panda3d.cxx @@ -143,14 +143,15 @@ thread_main() { } else { status = P3D_RC_generic_error; } + cerr << "Error getting URL " << _url << "\n"; + } else { + cerr << "Done getting URL " << _url << ", got " << bytes_sent << " bytes\n"; } P3D_instance_feed_url_stream (_instance, _unique_id, status, channel->get_status_code(), bytes_sent, NULL, 0); - - cerr << "Done getting URL " << _url << ", got " << bytes_sent << " bytes\n"; } bool