From 29c1d34cf6a51b3a5e4657c508cab2b8c86cbfc3 Mon Sep 17 00:00:00 2001 From: David Rose Date: Thu, 13 Aug 2009 08:18:15 +0000 Subject: [PATCH] serialize package download --- direct/src/plugin/p3dInstance.cxx | 149 +++++++++++++++++++---- direct/src/plugin/p3dInstance.h | 8 +- direct/src/plugin/p3dOsxSplashWindow.cxx | 43 ++++--- direct/src/plugin/p3dPackage.I | 45 +++++++ direct/src/plugin/p3dPackage.cxx | 93 ++++++++++++-- direct/src/plugin/p3dPackage.h | 15 ++- direct/src/plugin_npapi/ppInstance.cxx | 11 ++ direct/src/showutil/Packager.py | 71 ++++++----- 8 files changed, 346 insertions(+), 89 deletions(-) diff --git a/direct/src/plugin/p3dInstance.cxx b/direct/src/plugin/p3dInstance.cxx index 6eaadd0c53..200375a180 100644 --- a/direct/src/plugin/p3dInstance.cxx +++ b/direct/src/plugin/p3dInstance.cxx @@ -85,6 +85,8 @@ P3DInstance(P3D_request_ready_func *func, // Set some initial properties. _panda_script_object->set_float_property("downloadProgress", 0.0); + _panda_script_object->set_string_property("downloadPackageName", ""); + _panda_script_object->set_string_property("downloadPackageDisplayName", ""); _panda_script_object->set_bool_property("downloadComplete", false); _panda_script_object->set_string_property("status", "initial"); } @@ -108,7 +110,7 @@ P3DInstance:: // them. Packages::iterator pi; for (pi = _packages.begin(); pi != _packages.end(); ++pi) { - (*pi)->cancel_instance(this); + (*pi)->remove_instance(this); } _packages.clear(); @@ -628,13 +630,33 @@ add_package(P3DPackage *package) { } _packages.push_back(package); - package->set_instance(this); + package->add_instance(this); if (package->get_package_name() == "panda3d") { _panda3d = package; } } +//////////////////////////////////////////////////////////////////// +// Function: P3DInstance::get_packages_info_ready +// Access: Public +// Description: Returns true if all of the packages required by the +// instance have their information available and are +// ready to be downloaded, false if one or more of them +// is still waiting for information (or has failed). +//////////////////////////////////////////////////////////////////// +bool P3DInstance:: +get_packages_info_ready() const { + Packages::const_iterator pi; + for (pi = _packages.begin(); pi != _packages.end(); ++pi) { + if (!(*pi)->get_info_ready()) { + return false; + } + } + + return true; +} + //////////////////////////////////////////////////////////////////// // Function: P3DInstance::get_packages_ready // Access: Public @@ -1089,14 +1111,97 @@ make_splash_window() { } //////////////////////////////////////////////////////////////////// -// Function: P3DInstance::start_package_download +// Function: P3DInstance::report_package_info_ready // Access: Private -// Description: Notified when the package download begins. +// Description: Notified when a package information has been +// successfully downloaded and the package is idle, +// waiting for activate_download() to be called. //////////////////////////////////////////////////////////////////// void P3DInstance:: -start_package_download(P3DPackage *package) { - _panda_script_object->set_string_property("status", "downloading"); - send_notify("ondownloadbegin"); +report_package_info_ready(P3DPackage *package) { + if (get_packages_info_ready()) { + // All packages are ready to go. Let's start some download + // action. + _downloading_packages.clear(); + _total_download_size = 0; + Packages::const_iterator pi; + for (pi = _packages.begin(); pi != _packages.end(); ++pi) { + P3DPackage *package = (*pi); + if (package->get_info_ready() && !package->get_ready()) { + _downloading_packages.push_back(package); + _total_download_size += package->get_download_size(); + } + } + _download_package_index = 0; + _total_downloaded = 0; + + nout << "Beginning download of " << _downloading_packages.size() + << " packages, total " << _total_download_size + << " bytes required.\n"; + + _panda_script_object->set_string_property("status", "downloading"); + send_notify("ondownloadbegin"); + + start_next_download(); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DInstance::start_next_download +// Access: Private +// Description: Checks whether all packages are ready and waiting to +// be downloaded; if so, starts the next package in +// sequence downloading. +//////////////////////////////////////////////////////////////////// +void P3DInstance:: +start_next_download() { + while (_download_package_index < _downloading_packages.size()) { + P3DPackage *package = _downloading_packages[_download_package_index]; + if (package->get_failed()) { + // Too bad. TODO: fail. + return; + } + + if (!package->get_ready()) { + // This package is ready to download. Begin. + string name = package->get_package_display_name(); + if (name.empty()) { + name = package->get_package_name(); + } + _panda_script_object->set_string_property("downloadPackageName", package->get_package_name()); + _panda_script_object->set_string_property("downloadPackageDisplayName", name); + if (_splash_window != NULL) { + _splash_window->set_install_label("Installing " + name); + } + + nout << "Downloading " << package->get_package_name() + << ", package " << _download_package_index + 1 + << " of " << _downloading_packages.size() + << ", " << package->get_download_size() + << " bytes.\n"; + + package->activate_download(); + return; + } + + // This package has been downloaded. Move to the next. + _total_downloaded += package->get_download_size(); + ++_download_package_index; + } + + // Looks like we're all done downloading. Launch! + _downloading_packages.clear(); + + if (get_packages_ready()) { + _panda_script_object->set_bool_property("downloadComplete", true); + _panda_script_object->set_string_property("status", "starting"); + send_notify("ondownloadcomplete"); + + // Notify the session also. + if (_session != NULL) { + _session->report_packages_done(this, true); + } + } } //////////////////////////////////////////////////////////////////// @@ -1106,10 +1211,20 @@ start_package_download(P3DPackage *package) { //////////////////////////////////////////////////////////////////// void P3DInstance:: report_package_progress(P3DPackage *package, double progress) { + if (_download_package_index >= _downloading_packages.size() || + package != _downloading_packages[_download_package_index]) { + // Got a report from an unexpected package. + nout << "Got download progress report from " << package->get_package_name() + << ", not at download head (head is " << _download_package_index + << ")\n"; + return; + } + + // Scale the progress into the range appropriate to this package. + progress = (progress * package->get_download_size() + _total_downloaded) / _total_download_size; + progress = min(progress, 1.0); + if (_splash_window != NULL) { - if (!package->get_package_display_name().empty()) { - _splash_window->set_install_label("Installing " + package->get_package_display_name()); - } _splash_window->set_install_progress(progress); } _panda_script_object->set_float_property("downloadProgress", progress); @@ -1125,17 +1240,9 @@ void P3DInstance:: report_package_done(P3DPackage *package, bool success) { if (success) { report_package_progress(package, 1.0); - - if (get_packages_ready()) { - _panda_script_object->set_bool_property("downloadComplete", true); - _panda_script_object->set_string_property("status", "starting"); - send_notify("ondownloadcomplete"); - - // Notify the session also. - if (_session != NULL) { - _session->report_packages_done(this, success); - } - } + start_next_download(); + } else { + // TODO: fail. } } diff --git a/direct/src/plugin/p3dInstance.h b/direct/src/plugin/p3dInstance.h index 4efccaed29..5d8172d21e 100644 --- a/direct/src/plugin/p3dInstance.h +++ b/direct/src/plugin/p3dInstance.h @@ -80,6 +80,7 @@ public: inline P3D_request_ready_func *get_request_ready_func() const; void add_package(P3DPackage *package); + bool get_packages_info_ready() const; bool get_packages_ready() const; bool get_packages_failed() const; @@ -111,7 +112,8 @@ private: const string &property_name, P3D_object *value, bool needs_response, int unique_id); void make_splash_window(); - void start_package_download(P3DPackage *package); + void report_package_info_ready(P3DPackage *package); + void start_next_download(); void report_package_progress(P3DPackage *package, double progress); void report_package_done(P3DPackage *package, bool progress); @@ -160,6 +162,10 @@ private: typedef vector Packages; Packages _packages; + Packages _downloading_packages; + int _download_package_index; + size_t _total_download_size; + size_t _total_downloaded; // We keep the _panda3d pointer separately because it's so // important, but it's in the above vector also. diff --git a/direct/src/plugin/p3dOsxSplashWindow.cxx b/direct/src/plugin/p3dOsxSplashWindow.cxx index d46e0e44c9..15c5757534 100644 --- a/direct/src/plugin/p3dOsxSplashWindow.cxx +++ b/direct/src/plugin/p3dOsxSplashWindow.cxx @@ -240,27 +240,30 @@ paint_window() { &src_rect, &dest_rect, srcCopy, 0); } - int bar_width = min((int)(win_width * 0.6), 400); - int bar_height = min((int)(win_height * 0.1), 24); - int bar_x = (win_width - bar_width) / 2; - int bar_y = (win_height - bar_height * 2); - - int progress = bar_x + 1 + (int)((bar_width - 2) * _install_progress); - - Rect rbar = { bar_y, bar_x, bar_y + bar_height, bar_x + bar_width }; - Rect rneed = { bar_y + 1, progress, bar_y + bar_height - 1, bar_x + bar_width - 1 }; - Rect rdone = { bar_y + 1, bar_x + 1, bar_y + bar_height - 1, progress }; - FrameRect(&rbar); - - RGBColor blue = { 27756, 42405, 57568 }; - RGBForeColor(&blue); - PaintRect(&rdone); - EraseRect(&rneed); - - RGBColor black = { 0, 0, 0 }; - RGBForeColor(&black); - if (!_install_label.empty()) { + // Draw the progress bar. We don't draw this bar unless the + // install_label has been set nonempty. + int bar_width = min((int)(win_width * 0.6), 400); + int bar_height = min((int)(win_height * 0.1), 24); + int bar_x = (win_width - bar_width) / 2; + int bar_y = (win_height - bar_height * 2); + + int progress = bar_x + 1 + (int)((bar_width - 2) * _install_progress); + + Rect rbar = { bar_y, bar_x, bar_y + bar_height, bar_x + bar_width }; + Rect rneed = { bar_y + 1, progress, bar_y + bar_height - 1, bar_x + bar_width - 1 }; + Rect rdone = { bar_y + 1, bar_x + 1, bar_y + bar_height - 1, progress }; + FrameRect(&rbar); + + RGBColor blue = { 27756, 42405, 57568 }; + RGBForeColor(&blue); + PaintRect(&rdone); + EraseRect(&rneed); + + RGBColor black = { 0, 0, 0 }; + RGBForeColor(&black); + + // Now draw the install_label right above it. TextFont(0); TextFace(bold); TextMode(srcOr); diff --git a/direct/src/plugin/p3dPackage.I b/direct/src/plugin/p3dPackage.I index d29ee4db5b..578d273ceb 100755 --- a/direct/src/plugin/p3dPackage.I +++ b/direct/src/plugin/p3dPackage.I @@ -13,6 +13,51 @@ //////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::get_info_ready +// Access: Public +// Description: Returns true if the package file information has been +// downloaded and verified and is ready to be consulted, +// false if it is not yet available. +//////////////////////////////////////////////////////////////////// +inline bool P3DPackage:: +get_info_ready() const { + return _info_ready; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::get_download_size +// Access: Public +// Description: If get_info_ready() is true but get_ready() is false, +// it means the package is ready to be downloaded. In +// this case, this method returns the number of bytes +// that need to be downloaded for this package. This is +// intended to be used to estimate the download time for +// this package relative to other packages, for instance +// to update a progress bar sensibly. +//////////////////////////////////////////////////////////////////// +inline size_t P3DPackage:: +get_download_size() const { + return _download_size; +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::activate_download +// Access: Public +// Description: Authorizes the package to begin downloading and +// unpacking the meat of its data. Until this is +// called, the package will download its file +// information only, and then wait. +//////////////////////////////////////////////////////////////////// +inline void P3DPackage:: +activate_download() { + _allow_data_download = true; + + if (_info_ready) { + begin_data_download(); + } +} + //////////////////////////////////////////////////////////////////// // Function: P3DPackage::get_ready // Access: Public diff --git a/direct/src/plugin/p3dPackage.cxx b/direct/src/plugin/p3dPackage.cxx index f5d2c4d819..a54da05b53 100755 --- a/direct/src/plugin/p3dPackage.cxx +++ b/direct/src/plugin/p3dPackage.cxx @@ -53,6 +53,9 @@ P3DPackage(const string &package_name, _package_fullname += string("_") + _package_version; _package_dir += string("/") + _package_version; + _info_ready = false; + _download_size = 0; + _allow_data_download = false; _ready = false; _failed = false; _active_download = NULL; @@ -86,26 +89,26 @@ P3DPackage:: } //////////////////////////////////////////////////////////////////// -// Function: P3DPackage::set_instance +// Function: P3DPackage::add_instance // Access: Public // Description: Specifies an instance that may be responsible for // downloading this package. //////////////////////////////////////////////////////////////////// void P3DPackage:: -set_instance(P3DInstance *inst) { +add_instance(P3DInstance *inst) { _instances.push_back(inst); - begin_download(); + begin_info_download(); } //////////////////////////////////////////////////////////////////// -// Function: P3DPackage::cancel_instance +// Function: P3DPackage::remove_instance // Access: Public // Description: Indicates that the given instance will no longer be // responsible for downloading this package. //////////////////////////////////////////////////////////////////// void P3DPackage:: -cancel_instance(P3DInstance *inst) { +remove_instance(P3DInstance *inst) { assert(!_instances.empty()); if (inst == _instances[0]) { @@ -122,23 +125,25 @@ cancel_instance(P3DInstance *inst) { assert(ii != _instances.end()); _instances.erase(ii); - begin_download(); + begin_info_download(); } //////////////////////////////////////////////////////////////////// -// Function: P3DPackage::begin_download -// Access: Public -// Description: Begins downloading and installing the package, if -// needed. +// Function: P3DPackage::begin_info_download +// Access: Private +// Description: Begins downloading and installing the information +// about the package, including its file size and +// download source and such, if needed. This is +// generally a very small download. //////////////////////////////////////////////////////////////////// void P3DPackage:: -begin_download() { +begin_info_download() { if (_instances.empty()) { // Can't download without any instances. return; } - if (_ready) { + if (_info_ready) { // Already downloaded. return; } @@ -341,7 +346,49 @@ got_desc_file(TiXmlDocument *doc, bool freshly_downloaded) { // Great, we're ready to begin. report_done(true); - } else if (_uncompressed_archive.quick_verify(_package_dir)) { + } else { + // We need to get the file data still, but at least we know all + // about it by this point. + if (!_allow_data_download) { + // Not authorized to start downloading yet; just report that + // we're ready. + report_info_ready(); + } else { + // We've already been authorized to start downloading, so do it. + begin_data_download(); + } + } +} + +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::begin_data_download +// Access: Private +// Description: Begins downloading and installing the package data +// itself, if needed. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +begin_data_download() { + if (_instances.empty()) { + // Can't download without any instances. + return; + } + + if (_ready) { + // Already downloaded. + return; + } + + if (_active_download != NULL) { + // In the middle of downloading. + return; + } + + if (!_allow_data_download) { + // Not authorized yet. + return; + } + + if (_uncompressed_archive.quick_verify(_package_dir)) { // We need to re-extract the archive. extract_archive(); @@ -580,6 +627,25 @@ report_progress(double progress) { } } +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::report_info_ready +// Access: Private +// Description: Called when the package information has been +// successfully downloaded but activate_download() has +// not yet been called, and the package is now idle, +// waiting for activate_download() to be called. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +report_info_ready() { + _info_ready = true; + _download_size = _compressed_archive.get_size(); + + Instances::iterator ii; + for (ii = _instances.begin(); ii != _instances.end(); ++ii) { + (*ii)->report_package_info_ready(this); + } +} + //////////////////////////////////////////////////////////////////// // Function: P3DPackage::report_done // Access: Private @@ -590,6 +656,7 @@ report_progress(double progress) { void P3DPackage:: report_done(bool success) { if (success) { + _info_ready = true; _ready = true; _failed = false; } else { diff --git a/direct/src/plugin/p3dPackage.h b/direct/src/plugin/p3dPackage.h index 18d7c56931..7203a48dcb 100755 --- a/direct/src/plugin/p3dPackage.h +++ b/direct/src/plugin/p3dPackage.h @@ -41,6 +41,10 @@ public: const string &package_version); ~P3DPackage(); + inline bool get_info_ready() const; + inline size_t get_download_size() const; + + inline void activate_download(); inline bool get_ready() const; inline bool get_failed() const; inline const string &get_package_dir() const; @@ -48,8 +52,8 @@ public: inline const string &get_package_version() const; inline const string &get_package_display_name() const; - void set_instance(P3DInstance *inst); - void cancel_instance(P3DInstance *inst); + void add_instance(P3DInstance *inst); + void remove_instance(P3DInstance *inst); private: enum DownloadType { @@ -71,7 +75,7 @@ private: DownloadType _dtype; }; - void begin_download(); + void begin_info_download(); void download_contents_file(); void contents_file_download_finished(bool success); @@ -79,6 +83,7 @@ private: void desc_file_download_finished(bool success); void got_desc_file(TiXmlDocument *doc, bool freshly_downloaded); + void begin_data_download(); void download_compressed_archive(bool allow_partial); void compressed_archive_download_progress(double progress); void compressed_archive_download_finished(bool success); @@ -87,6 +92,7 @@ private: void extract_archive(); void report_progress(double progress); + void report_info_ready(); void report_done(bool success); void start_download(DownloadType dtype, const string &url, const string &pathname, bool allow_partial); @@ -106,6 +112,9 @@ private: string _desc_file_basename; string _desc_file_pathname; + bool _info_ready; + size_t _download_size; + bool _allow_data_download; bool _ready; bool _failed; Download *_active_download; diff --git a/direct/src/plugin_npapi/ppInstance.cxx b/direct/src/plugin_npapi/ppInstance.cxx index eadaa2f6e4..2fc22a3429 100644 --- a/direct/src/plugin_npapi/ppInstance.cxx +++ b/direct/src/plugin_npapi/ppInstance.cxx @@ -27,6 +27,10 @@ #include #include // strcmp() +#ifndef _WIN32 +#include +#endif + PPInstance::FileDatas PPInstance::_file_datas; //////////////////////////////////////////////////////////////////// @@ -1336,6 +1340,13 @@ thread_run() { // So far, so good. Read some more. _file.read(buffer, buffer_size); count = _file.gcount(); + + // This is useful for development, to slow things down enough to + // see the progress bar move. + struct timeval tv; + tv.tv_sec = 0; + tv.tv_usec = 10000; + select(0, NULL, NULL, NULL, &tv); } // End of file. diff --git a/direct/src/showutil/Packager.py b/direct/src/showutil/Packager.py index 5508094f7e..aed3209e4f 100644 --- a/direct/src/showutil/Packager.py +++ b/direct/src/showutil/Packager.py @@ -67,41 +67,14 @@ class Packager: def close(self): """ Writes out the contents of the current package. """ - if not self.p3dApplication and not self.version: - # We must have a version string for packages. - self.version = '0.0' - - self.packageBasename = self.packageName - packageDir = self.packageName - if self.platform: - self.packageBasename += '_' + self.platform - packageDir += '/' + self.platform - if self.version: - self.packageBasename += '_' + self.version - packageDir += '/' + self.version - - self.packageDesc = self.packageBasename + '.xml' - self.packageImportDesc = self.packageBasename + '_import.xml' - if self.p3dApplication: - self.packageBasename += '.p3d' - packageDir = '' - else: - self.packageBasename += '.mf' - packageDir += '/' - - self.packageFilename = packageDir + self.packageBasename - self.packageDesc = packageDir + self.packageDesc - self.packageImportDesc = packageDir + self.packageImportDesc - - self.packageFullpath = Filename(self.packager.installDir, self.packageFilename) - self.packageFullpath.makeDir() - self.packageFullpath.unlink() - if self.dryRun: self.multifile = None else: + # Write the multifile to a temporary filename until we + # know enough to determine the output filename. + multifileFilename = Filename.temporary('', self.packageName) self.multifile = Multifile() - self.multifile.openReadWrite(self.packageFullpath) + self.multifile.openReadWrite(multifileFilename) self.extracts = [] self.components = [] @@ -181,6 +154,40 @@ class Packager: xmodule.SetAttribute('name', moduleName) self.components.append(xmodule) + # Now that we've processed all of the component files, + # (and set our platform if necessary), we can generate the + # output filename and write the output files. + + if not self.p3dApplication and not self.version: + # We must have a version string for packages. + self.version = '0.0' + + self.packageBasename = self.packageName + packageDir = self.packageName + if self.platform: + self.packageBasename += '_' + self.platform + packageDir += '/' + self.platform + if self.version: + self.packageBasename += '_' + self.version + packageDir += '/' + self.version + + self.packageDesc = self.packageBasename + '.xml' + self.packageImportDesc = self.packageBasename + '_import.xml' + if self.p3dApplication: + self.packageBasename += '.p3d' + packageDir = '' + else: + self.packageBasename += '.mf' + packageDir += '/' + + self.packageFilename = packageDir + self.packageBasename + self.packageDesc = packageDir + self.packageDesc + self.packageImportDesc = packageDir + self.packageImportDesc + + self.packageFullpath = Filename(self.packager.installDir, self.packageFilename) + self.packageFullpath.makeDir() + self.packageFullpath.unlink() + if not self.dryRun: self.freezer.addToMultifile(self.multifile, self.compressionLevel) if self.p3dApplication: @@ -188,6 +195,8 @@ class Packager: self.multifile.repack() self.multifile.close() + multifileFilename.renameTo(self.packageFullpath) + if not self.p3dApplication: self.compressMultifile() self.writeDescFile()