From bf98a3c9a9f1a2f9e607f30207a82101889f7d09 Mon Sep 17 00:00:00 2001 From: David Rose Date: Mon, 4 Nov 2002 21:22:13 +0000 Subject: [PATCH] fixes for new publish script, patch fixes --- panda/src/downloader/downloadDb.cxx | 159 +++++++--- panda/src/downloader/downloadDb.h | 20 +- panda/src/downloadertools/Sources.pp | 9 + panda/src/downloadertools/build_patch.cxx | 17 +- panda/src/downloadertools/check_md5.cxx | 33 +- panda/src/downloadertools/pcompress.cxx | 23 +- panda/src/downloadertools/show_ddb.cxx | 37 +++ panda/src/express/crypto_utils.cxx | 2 + panda/src/express/hashVal.I | 42 +-- panda/src/express/hashVal.cxx | 11 + panda/src/express/hashVal.h | 10 +- panda/src/express/patchfile.I | 42 ++- panda/src/express/patchfile.cxx | 352 +++++++++++++++------- panda/src/express/patchfile.h | 29 +- 14 files changed, 552 insertions(+), 234 deletions(-) create mode 100644 panda/src/downloadertools/show_ddb.cxx diff --git a/panda/src/downloader/downloadDb.cxx b/panda/src/downloader/downloadDb.cxx index 395cdb3c20..7197e266e6 100644 --- a/panda/src/downloader/downloadDb.cxx +++ b/panda/src/downloader/downloadDb.cxx @@ -273,8 +273,8 @@ expand_client_multifile(string mfname) { //////////////////////////////////////////////////////////////////// -// Function: DownloadDb:: -// Access: Public +// Function: DownloadDb::read_db +// Access: Published // Description: //////////////////////////////////////////////////////////////////// DownloadDb::Db DownloadDb:: @@ -310,8 +310,8 @@ read_db(Filename &file, bool want_server_info) { } //////////////////////////////////////////////////////////////////// -// Function: DownloadDb:: -// Access: Public +// Function: DownloadDb::read_db +// Access: Published // Description: //////////////////////////////////////////////////////////////////// DownloadDb::Db DownloadDb:: @@ -337,8 +337,8 @@ read_db(Ramfile &file, bool want_server_info) { //////////////////////////////////////////////////////////////////// -// Function: DownloadDb:: -// Access: Public +// Function: DownloadDb::write_db +// Access: Published // Description: //////////////////////////////////////////////////////////////////// bool DownloadDb:: @@ -1075,75 +1075,146 @@ output(ostream &out) const { //////////////////////////////////////////////////////////////////// // Function: DownloadDb::add_version -// Access: Public -// Description: Note: version numbers start at 1 +// Access: Published +// Description: Appends a new version of the file onto the end of the +// list, or changes the hash associated with a version +// previously added. +// +// Note: version numbers start at 1 //////////////////////////////////////////////////////////////////// void DownloadDb:: -add_version(const Filename &name, HashVal hash, Version version) { +add_version(const Filename &name, const HashVal &hash, int version) { nassertv(version >= 1); - // Try to find this name in the map - VersionMap::iterator i = _versions.find(name); + VectorHash &vhash = _versions[name]; + int size = vhash.size(); + + // We should not skip over versions as we add them. + nassertv(version <= size+1); + + if (version-1 < size) { + // If you are overwriting an old hash value, just rewrite the value + vhash[version-1] = hash; - // If we did not find it, put a new vectorHash at this name_code - if (i == _versions.end()) { - vectorHash v; - nassertv(version == 1); - v.push_back(hash); - _versions[name] = v; } else { - int size = (*i).second.size(); - - // Assert that this version is less than or equal to next version in the list - nassertv(version<=size+1); - - // If you are overwriting an old hash value, just insert the new value - if (version-1 < size) { - (*i).second[version-1] = hash; - } else { - // add this hash at the end of the vector - (*i).second.push_back(hash); - } + // Otherwise, extend the vector. + vhash.push_back(hash); } } +//////////////////////////////////////////////////////////////////// +// Function: DownloadDb::insert_new_version +// Access: Published +// Description: Inserts a new version 1 copy of the file, sliding all +// the other versions up by one. +//////////////////////////////////////////////////////////////////// +void DownloadDb:: +insert_new_version(const Filename &name, const HashVal &hash) { + VectorHash &vhash = _versions[name]; + vhash.insert(vhash.begin(), hash); +} + //////////////////////////////////////////////////////////////////// // Function: DownloadDb::has_version -// Access: Public +// Access: Published // Description: Returns true if the indicated file has version // information, false otherwise. Some files recorded in // the database may not bother to track versions. //////////////////////////////////////////////////////////////////// bool DownloadDb:: -has_version(const Filename &name) { +has_version(const Filename &name) const { return (_versions.find(name) != _versions.end()); } +//////////////////////////////////////////////////////////////////// +// Function: DownloadDb::get_num_versions +// Access: Published +// Description: Returns the number of versions stored for the +// indicated file. +//////////////////////////////////////////////////////////////////// +int DownloadDb:: +get_num_versions(const Filename &name) const { + VersionMap::const_iterator vmi = _versions.find(name); + if (vmi == _versions.end()) { + return 0; + } + + return (int)(*vmi).second.size(); +} + +//////////////////////////////////////////////////////////////////// +// Function: DownloadDb::set_num_versions +// Access: Published +// Description: Reduces the number of versions of a particular file +// stored in the ddb by throwing away all versions +// higher than the indicated index. +//////////////////////////////////////////////////////////////////// +void DownloadDb:: +set_num_versions(const Filename &name, int num_versions) { + VersionMap::iterator vmi = _versions.find(name); + if (vmi == _versions.end()) { + nassertv(num_versions == 0); + return; + } + + VectorHash &vhash = (*vmi).second; + + nassertv(num_versions <= (int)vhash.size()); + vhash.erase(vhash.begin() + num_versions, vhash.end()); +} + //////////////////////////////////////////////////////////////////// // Function: DownloadDb::get_version -// Access: Public +// Access: Published // Description: Returns the version number of this particular file, // determined by looking up the hash generated from the // file. Returns -1 if the version number cannot be // determined. //////////////////////////////////////////////////////////////////// int DownloadDb:: -get_version(const Filename &name, HashVal hash) { +get_version(const Filename &name, const HashVal &hash) const { VersionMap::const_iterator vmi = _versions.find(name); if (vmi == _versions.end()) { downloader_cat.debug() << "DownloadDb::get_version() - can't find: " << name << endl; return -1; } - vectorHash ulvec = (*vmi).second; - vectorHash::iterator i = find(ulvec.begin(), ulvec.end(), hash); - if (i != ulvec.end()) - return (i - ulvec.begin() + 1); + const VectorHash &vhash = (*vmi).second; + VectorHash::const_iterator i = find(vhash.begin(), vhash.end(), hash); + if (i != vhash.end()) + return (i - vhash.begin() + 1); downloader_cat.debug() << "DownloadDb::get_version() - can't find hash: " << hash << endl; return -1; } +//////////////////////////////////////////////////////////////////// +// Function: DownloadDb::get_hash +// Access: Published +// Description: Returns the MD5 hash associated with the indicated +// version of the indicated file. +//////////////////////////////////////////////////////////////////// +const HashVal &DownloadDb:: +get_hash(const Filename &name, int version) const { + static HashVal bogus_hash; + + VersionMap::const_iterator vmi = _versions.find(name); + if (vmi == _versions.end()) { + downloader_cat.error() + << "DownloadDb::get_hash() - can't find: " << name << endl; + return bogus_hash; + } + + const VectorHash &vhash = (*vmi).second; + if (version < 1 || version > (int)vhash.size()) { + downloader_cat.error() + << "DownloadDb::get_hash() - no version " << version + << " for " << name << endl; + return bogus_hash; + } + return vhash[version - 1]; +} + //////////////////////////////////////////////////////////////////// // Function: DownloadDb::write_version_map // Access: Protected @@ -1154,7 +1225,7 @@ write_version_map(ofstream &write_stream) { _master_datagram.clear(); VersionMap::iterator vmi; - vectorHash::iterator i; + VectorHash::iterator i; string name; HashVal hash; @@ -1259,18 +1330,14 @@ read_version_map(istream &read_stream) { //////////////////////////////////////////////////////////////////// void DownloadDb:: output_version_map(ostream &out) const { - out << " Version Map: " << endl; + out << "Version Map: " << endl; VersionMap::const_iterator vmi; - vectorHash::const_iterator i; + VectorHash::const_iterator i; for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) { - out << " Filename: " << (*vmi).first; + out << " " << (*vmi).first << endl; for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) { HashVal hash = *i; - out << " [" << hash.get_value(0) - << " " << hash.get_value(1) - << " " << hash.get_value(2) - << " " << hash.get_value(3) - << "]" << endl; + out << " " << hash << endl; } } out << endl; diff --git a/panda/src/downloader/downloadDb.h b/panda/src/downloader/downloadDb.h index 32d1056039..c858b59ada 100644 --- a/panda/src/downloader/downloadDb.h +++ b/panda/src/downloader/downloadDb.h @@ -52,7 +52,6 @@ A Db is a Vector MultifileRecord is a Vector */ -typedef int Version; typedef float Phase; //////////////////////////////////////////////////////////////////// @@ -202,21 +201,26 @@ PUBLISHED: bool write_db(Filename &file, Db db, bool want_server_info); public: - // The download db stores two databases, one that represents the client's state - // and one that represents the server state + // The download db stores two databases, one that represents the + // client's state and one that represents the server state. Db _client_db; Db _server_db; // Magic number for knowing this is a download Db static PN_uint32 _magic_number; static PN_uint32 _bogus_magic_number; - typedef pvector vectorHash; - typedef pmap VersionMap; + typedef pvector VectorHash; + typedef pmap VersionMap; PUBLISHED: - void add_version(const Filename &name, HashVal hash, Version version); - bool has_version(const Filename &name); - int get_version(const Filename &name, HashVal hash); + void add_version(const Filename &name, const HashVal &hash, int version); + void insert_new_version(const Filename &name, const HashVal &hash); + bool has_version(const Filename &name) const; + int get_num_versions(const Filename &name) const; + void set_num_versions(const Filename &name, int num_versions); + + int get_version(const Filename &name, const HashVal &hash) const; + const HashVal &get_hash(const Filename &name, int version) const; protected: void write_version_map(ofstream &write_stream); diff --git a/panda/src/downloadertools/Sources.pp b/panda/src/downloadertools/Sources.pp index 2dcd1fb7b5..aec83d2644 100644 --- a/panda/src/downloadertools/Sources.pp +++ b/panda/src/downloadertools/Sources.pp @@ -22,6 +22,15 @@ #end bin_target +#begin bin_target + #define TARGET show_ddb + #define BUILD_TARGET $[HAVE_CRYPTO] + + #define SOURCES \ + show_ddb.cxx + +#end bin_target + #begin bin_target #define TARGET check_adler #define BUILD_TARGET $[HAVE_ZLIB] diff --git a/panda/src/downloadertools/build_patch.cxx b/panda/src/downloadertools/build_patch.cxx index 1515f81ca9..5801e4824b 100644 --- a/panda/src/downloadertools/build_patch.cxx +++ b/panda/src/downloadertools/build_patch.cxx @@ -16,14 +16,14 @@ // //////////////////////////////////////////////////////////////////// -#include +#include "pandabase.h" #ifndef HAVE_GETOPT - #include + #include "gnu_getopt.h" #else #include #endif -#include -#include +#include "patchfile.h" +#include "filename.h" int main(int argc, char *argv[]) { @@ -32,17 +32,18 @@ main(int argc, char *argv[]) { return 1; } - Filename src_file = argv[1]; + Filename src_file = Filename::from_os_specific(argv[1]); src_file.set_binary(); - Filename dest_file = argv[2]; + Filename dest_file = Filename::from_os_specific(argv[2]); dest_file.set_binary(); + Filename patch_file = dest_file.get_fullpath() + ".pch"; Patchfile pfile; cerr << "Building patch file to convert " << src_file << " to " - << dest_file << endl; - if (pfile.build(src_file, dest_file) == false) { + << dest_file << endl; + if (pfile.build(src_file, dest_file, patch_file) == false) { cerr << "build patch failed" << endl; return 1; } diff --git a/panda/src/downloadertools/check_md5.cxx b/panda/src/downloadertools/check_md5.cxx index 1a62c8972c..7b4722b8cb 100644 --- a/panda/src/downloadertools/check_md5.cxx +++ b/panda/src/downloadertools/check_md5.cxx @@ -16,32 +16,20 @@ // //////////////////////////////////////////////////////////////////// -#include -#include -#include +#include "crypto_utils.h" +#include "hashVal.h" #include "filename.h" int main(int argc, char *argv[]) { - const char *usagestr="Usage: check_md5 [-dbfmt_output] "; + const char *usagestr = "Usage: check_md5 "; if (argc < 2) { cerr << usagestr << endl; return 1; } - bool bRemoveBrackets = (strcmp("-dbfmt_output",argv[1])==0); - Filename source_file; - - if(bRemoveBrackets) { - if(argc<3) { - cerr << usagestr << endl; - return 1; - } - source_file = Filename::from_os_specific(argv[2]); - } else { - source_file = Filename::from_os_specific(argv[1]); - } + source_file = Filename::from_os_specific(argv[1]); if(!source_file.exists()) { cerr << usagestr << endl; @@ -52,13 +40,10 @@ main(int argc, char *argv[]) { HashVal hash; md5_a_file(source_file, hash); - if(bRemoveBrackets) { - hash.set_output_brackets(false); - // output base of Filename along w/md5 - cout << source_file.get_basename() << " "; - } - - cout << hash << endl; - + // output base of Filename along w/md5 + cout << source_file.get_basename() << " "; + hash.output(cout); + cout << endl; + return 0; } diff --git a/panda/src/downloadertools/pcompress.cxx b/panda/src/downloadertools/pcompress.cxx index 740643b0f0..82f70745a1 100644 --- a/panda/src/downloadertools/pcompress.cxx +++ b/panda/src/downloadertools/pcompress.cxx @@ -23,14 +23,20 @@ int main(int argc, char *argv[]) { if (argc < 2) { - cerr << "Usage: pcompress " << endl; + cerr << "Usage: pcompress []" << endl; return 1; } - Filename source_file = argv[1]; - string dname = argv[1]; - dname += ".pz"; - Filename dest_file = dname; + bool implicit_dest_file; + Filename source_file = Filename::from_os_specific(argv[1]); + Filename dest_file; + if (argc < 3) { + dest_file = source_file.get_fullpath() + ".pz"; + implicit_dest_file = true; + } else { + dest_file = Filename::from_os_specific(argv[2]); + implicit_dest_file = false; + } // Open source file ifstream read_stream; @@ -53,7 +59,7 @@ main(int argc, char *argv[]) { // Open destination file ofstream write_stream; dest_file.set_binary(); - if (!dest_file.open_write(write_stream)) { + if (!dest_file.open_write(write_stream, true)) { cerr << "failed to open: " << dest_file << endl; return 1; } @@ -70,7 +76,10 @@ main(int argc, char *argv[]) { read_stream.close(); write_stream.close(); - source_file.unlink(); + + if (implicit_dest_file) { + source_file.unlink(); + } return 0; } diff --git a/panda/src/downloadertools/show_ddb.cxx b/panda/src/downloadertools/show_ddb.cxx new file mode 100644 index 0000000000..55f499067d --- /dev/null +++ b/panda/src/downloadertools/show_ddb.cxx @@ -0,0 +1,37 @@ +// Filename: show_ddb.cxx +// Created by: drose (02Nov02) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://www.panda3d.org/license.txt . +// +// To contact the maintainers of this program write to +// panda3d@yahoogroups.com . +// +//////////////////////////////////////////////////////////////////// + +#include "pandabase.h" +#include "downloadDb.h" +#include "filename.h" + +int +main(int argc, char *argv[]) { + if (argc != 3) { + cerr << "Usage: show_ddb server.ddb client.ddb\n"; + return 1; + } + + Filename server_ddb = Filename::from_os_specific(argv[1]); + Filename client_ddb = Filename::from_os_specific(argv[2]); + + DownloadDb db(server_ddb, client_ddb); + db.output_version_map(cout); + + return 0; +} diff --git a/panda/src/express/crypto_utils.cxx b/panda/src/express/crypto_utils.cxx index f6a5d430b5..8da2d7b093 100644 --- a/panda/src/express/crypto_utils.cxx +++ b/panda/src/express/crypto_utils.cxx @@ -79,6 +79,8 @@ read32(istream& is) { void md5_a_file(const Filename &name, HashVal &ret) { + nassertv(name.exists()); + ostringstream os; MD5 md5; diff --git a/panda/src/express/hashVal.I b/panda/src/express/hashVal.I index 39b8019b3e..aa87a5daa7 100644 --- a/panda/src/express/hashVal.I +++ b/panda/src/express/hashVal.I @@ -19,18 +19,30 @@ //////////////////////////////////////////////////////////////////// // Function: HashVal::Constructor -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// INLINE HashVal:: -HashVal(void) { +HashVal() { hv[0] = hv[1] = hv[2] = hv[3] = 0; - _bUseBrackets = true; +} + +//////////////////////////////////////////////////////////////////// +// Function: HashVal::Copy Constructor +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +INLINE HashVal:: +HashVal(const HashVal ©) { + hv[0] = copy.hv[0]; + hv[1] = copy.hv[1]; + hv[2] = copy.hv[2]; + hv[3] = copy.hv[3]; } //////////////////////////////////////////////////////////////////// // Function: HashVal::operator == -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// INLINE bool HashVal:: @@ -43,7 +55,7 @@ operator == (const HashVal &other) const { //////////////////////////////////////////////////////////////////// // Function: HashVal::operator != -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// INLINE bool HashVal:: @@ -53,7 +65,7 @@ operator != (const HashVal &other) const { //////////////////////////////////////////////////////////////////// // Function: HashVal::get_value -// Access: Public +// Access: Published // Description: Returns the integer value of the indicated component. //////////////////////////////////////////////////////////////////// INLINE uint HashVal:: @@ -65,7 +77,7 @@ get_value(int val) const { //////////////////////////////////////////////////////////////////// // Function: HashVal::set_value -// Access: Public +// Access: Published // Description: Sets the hash value at index val //////////////////////////////////////////////////////////////////// INLINE void HashVal:: @@ -77,21 +89,11 @@ set_value(int val, uint hashval) { //////////////////////////////////////////////////////////////////// // Function: HashVal::output -// Access: Public -// Description: +// Access: Published +// Description: The output method does not itself output enclosing +// brackets, but the ostream operator << does. //////////////////////////////////////////////////////////////////// INLINE void HashVal:: output(ostream &out) const { - if(_bUseBrackets) - out << "["; out << hv[0] << " " << hv[1] << " " << hv[2] << " " << hv[3]; - if(_bUseBrackets) - out << "]"; } - -INLINE void HashVal:: -set_output_brackets(bool bUseBrackets) { - _bUseBrackets = bUseBrackets; -} - - diff --git a/panda/src/express/hashVal.cxx b/panda/src/express/hashVal.cxx index fc8ddab1dc..bc1f9e5f4b 100644 --- a/panda/src/express/hashVal.cxx +++ b/panda/src/express/hashVal.cxx @@ -19,3 +19,14 @@ #include "hashVal.h" +//////////////////////////////////////////////////////////////////// +// Function: HashVal::as_string +// Access: Published +// Description: Returns the HashVal as a string with four numbers. +//////////////////////////////////////////////////////////////////// +string HashVal:: +as_string() const { + ostringstream strm; + output(strm); + return strm.str(); +} diff --git a/panda/src/express/hashVal.h b/panda/src/express/hashVal.h index cd508f5e0c..7f9a2a5e43 100644 --- a/panda/src/express/hashVal.h +++ b/panda/src/express/hashVal.h @@ -30,19 +30,23 @@ class EXPCL_PANDAEXPRESS HashVal { PUBLISHED: INLINE HashVal(); + INLINE HashVal(const HashVal ©); + INLINE bool operator == (const HashVal &other) const; INLINE bool operator != (const HashVal &other) const; INLINE uint get_value(int val) const; INLINE void set_value(int val, uint hash); INLINE void output(ostream &out) const; - uint hv[4]; + string as_string() const; + public: - INLINE void set_output_brackets(bool bUseBrackets); - bool _bUseBrackets; + uint hv[4]; }; INLINE ostream &operator << (ostream &out, const HashVal &hv) { + out << "["; hv.output(out); + out << "]"; return out; } diff --git a/panda/src/express/patchfile.I b/panda/src/express/patchfile.I index 8225aac77b..5a90c6d529 100644 --- a/panda/src/express/patchfile.I +++ b/panda/src/express/patchfile.I @@ -20,7 +20,7 @@ //////////////////////////////////////////////////////////////////// // Function: Patchfile::get_progress -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// INLINE float Patchfile:: @@ -36,7 +36,7 @@ get_progress(void) const { //////////////////////////////////////////////////////////////////// // Function: Patchfile::set_footprint_length -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// INLINE void Patchfile:: @@ -47,7 +47,7 @@ set_footprint_length(int length) { //////////////////////////////////////////////////////////////////// // Function: Patchfile::get_footprint_length -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// INLINE int Patchfile:: @@ -57,10 +57,44 @@ get_footprint_length() { //////////////////////////////////////////////////////////////////// // Function: Patchfile::reset_footprint_length -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// INLINE void Patchfile:: reset_footprint_length() { _footprint_length = _DEFAULT_FOOTPRINT_LENGTH; } + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::has_source_hash +// Access: Published +// Description: Returns true if the MD5 hash for the source file is +// known. (Some early versions of the patch file did +// not store this information.) +//////////////////////////////////////////////////////////////////// +INLINE bool Patchfile:: +has_source_hash() const { + return (_version_number >= 1); +} + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::get_source_hash +// Access: Published +// Description: Returns the MD5 hash for the source file. +//////////////////////////////////////////////////////////////////// +INLINE const HashVal &Patchfile:: +get_source_hash() const { + nassertr(has_source_hash(), _MD5_ofSource); + return _MD5_ofSource; +} + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::get_result_hash +// Access: Published +// Description: Returns the MD5 hash for the file after the patch has +// been applied. +//////////////////////////////////////////////////////////////////// +INLINE const HashVal &Patchfile:: +get_result_hash() const { + return _MD5_ofResult; +} diff --git a/panda/src/express/patchfile.cxx b/panda/src/express/patchfile.cxx index d68b45ac27..b4339b92b4 100644 --- a/panda/src/express/patchfile.cxx +++ b/panda/src/express/patchfile.cxx @@ -50,16 +50,26 @@ ClockObject *globalClock = ClockObject::get_global_clock(); ///// IF THIS CHANGES, UPDATE installerApplyPatch.cxx IN THE INSTALLER //////////////////////////////////////////////////////////////////// // [ HEADER ] -// 4 bytes 0xfeebfaab ("magic number") +// 4 bytes 0xfeebfaac ("magic number") +// (older patch files have a magic number 0xfeebfaab, +// indicating they are version number 0.) +// 2 bytes version number (if magic number == 0xfeebfaac) +// 4 bytes length of starting file (if version >= 1) +// 16 bytes MD5 of starting file (if version >= 1) // 4 bytes length of resulting patched file // 16 bytes MD5 of resultant patched file -const int _header_length = sizeof(PN_uint32) + sizeof(PN_uint32) + (4*sizeof(PN_uint32)); + +const int _v0_header_length = 4 + 4 + 16; +const int _v1_header_length = 4 + 2 + 4 + 16 + 4 + 16; // // [ ADD/COPY pairs; repeated N times ] // 2 bytes AL = ADD length // AL bytes bytes to add // 2 bytes CL = COPY length -// 4 bytes offset of data to copy from original file, if CL != 0 +// 4 bytes offset of data to copy from original file, if CL != 0. +// If version >= 2, offset is relative to end of previous +// copy block; if version < 2, offset is relative to +// beginning of file. // // [ TERMINATOR ] // 2 bytes zero-length ADD @@ -70,7 +80,12 @@ const int _header_length = sizeof(PN_uint32) + sizeof(PN_uint32) + (4*sizeof(PN_ //////////////////////////////////////////////////////////////////// // Defines //////////////////////////////////////////////////////////////////// -const PN_uint32 Patchfile::_magic_number = 0xfeebfaab; +const PN_uint32 Patchfile::_v0_magic_number = 0xfeebfaab; +const PN_uint32 Patchfile::_magic_number = 0xfeebfaac; + +// Created version 1 on 11/2/02 to store length and MD5 of original file. +// To version 2 on 11/2/02 to store copy offsets as relative. +const PN_uint16 Patchfile::_current_version = 2; const PN_uint32 Patchfile::_HASHTABLESIZE = PN_uint32(1) << 16; const PN_uint32 Patchfile::_DEFAULT_FOOTPRINT_LENGTH = 9; // this produced the smallest patch file for libpanda.dll when tested, 12/20/2000 @@ -108,6 +123,8 @@ init(PT(Buffer) buffer) { _initiated = false; nassertv(!buffer.is_null()); _buffer = buffer; + + _version_number = 0; reset_footprint_length(); } @@ -159,28 +176,19 @@ cleanup(void) { //////////////////////////////////////////////////////////////////// // Function: Patchfile::initiate -// Access: Public +// Access: Published // Description: Set up to apply the patch to the file (original // file and patch are destroyed in the process). //////////////////////////////////////////////////////////////////// int Patchfile:: -initiate(Filename &patch_file, Filename &file) { - if (true == _initiated) { +initiate(const Filename &patch_file, const Filename &file) { + if (_initiated) { express_cat.error() << "Patchfile::initiate() - Patching has already been initiated" << endl; return EU_error_abort; } - // Open the patch file for read - _patch_file = patch_file; - _patch_file.set_binary(); - if (!_patch_file.open_read(_patch_stream)) { - express_cat.error() - << "Patchfile::initiate() - Failed to open file: " << _patch_file << endl; - return get_write_error(); - } - // Open the original file for read _orig_file = file; _orig_file.set_binary(); @@ -215,42 +223,39 @@ initiate(Filename &patch_file, Filename &file) { << "Using temporary file " << _temp_file << "\n"; } - ///////////// - // read header, make sure the patch file is valid - - StreamReader patch_reader(_patch_stream); - - // check the magic number - nassertr(_buffer->get_length() >= _header_length, false); - PN_uint32 magic_number = patch_reader.get_uint32(); - if (magic_number != _magic_number) { - express_cat.error() - << "Patchfile::initiate() - invalid patch file: " << _patch_file << endl; - return EU_error_file_invalid; - } - - // get the length of the patched result file - _result_file_length = patch_reader.get_uint32(); - - // get the MD5 of the resultant patched file - _MD5_ofResult.set_value(0, patch_reader.get_uint32()); - _MD5_ofResult.set_value(1, patch_reader.get_uint32()); - _MD5_ofResult.set_value(2, patch_reader.get_uint32()); - _MD5_ofResult.set_value(3, patch_reader.get_uint32()); - - express_cat.debug() - << "Patchfile::initiate() - valid patchfile" << endl; + int result = internal_read_header(patch_file); _total_bytes_processed = 0; _initiated = true; + return result; +} - return EU_success; +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::read_header +// Access: Published +// Description: Opens the patch file for reading, and gets the header +// information from the file but does not begin to do +// any real work. This can be used to query the data +// stored in the patch. +//////////////////////////////////////////////////////////////////// +int Patchfile:: +read_header(const Filename &patch_file) { + if (_initiated) { + express_cat.error() + << "Patchfile::initiate() - Patching has already been initiated" + << endl; + return EU_error_abort; + } + + int result = internal_read_header(patch_file); + _patch_stream.close(); + return result; } //////////////////////////////////////////////////////////////////// // Function: Patchfile::run -// Access: Public +// Access: Published // Description: Perform one buffer's worth of patching //////////////////////////////////////////////////////////////////// int Patchfile:: @@ -260,7 +265,7 @@ run(void) { int bytes_read; PN_uint16 ADD_length; PN_uint16 COPY_length; - PN_uint32 COPY_offset; + PN_int32 COPY_offset; if (_initiated == false) { express_cat.error() @@ -284,20 +289,18 @@ run(void) { _total_bytes_processed += (int)ADD_length; // if there are bytes to add, read them from patch file and write them to output - if (0 != ADD_length) { - PN_uint32 bytes_left = (PN_uint32)ADD_length; + if (express_cat.is_spam()) { + express_cat.spam() + << "ADD: " << ADD_length << " (to " + << _write_stream.tellp() << ")" << endl; + } - if (express_cat.is_spam()) { - express_cat.spam() - << "ADD: " << ADD_length << endl; - } - - while (bytes_left > 0) { - PN_uint32 bytes_this_time = (PN_uint32) min(bytes_left, (PN_uint32) buflen); - _patch_stream.read(_buffer->_buffer, bytes_this_time); - _write_stream.write(_buffer->_buffer, bytes_this_time); - bytes_left -= bytes_this_time; - } + PN_uint32 bytes_left = (PN_uint32)ADD_length; + while (bytes_left > 0) { + PN_uint32 bytes_this_time = (PN_uint32) min(bytes_left, (PN_uint32) buflen); + _patch_stream.read(_buffer->_buffer, bytes_this_time); + _write_stream.write(_buffer->_buffer, bytes_this_time); + bytes_left -= bytes_this_time; } /////////// @@ -312,17 +315,23 @@ run(void) { if (0 != COPY_length) { // read copy offset nassertr(_buffer->get_length() >= (int)sizeof(COPY_offset), false); - COPY_offset = patch_reader.get_uint32(); + COPY_offset = patch_reader.get_int32(); + + // seek to the copy source pos + if (_version_number < 2) { + _origfile_stream.seekg(COPY_offset, ios::beg); + } else { + _origfile_stream.seekg(COPY_offset, ios::cur); + } if (express_cat.is_spam()) { express_cat.spam() - << "COPY: " << COPY_length << " bytes at offset " << COPY_offset + << "COPY: " << COPY_length << " bytes from offset " + << COPY_offset << " (from " << _origfile_stream.tellg() + << " to " << _write_stream.tellp() << ")" << endl; } - // seek to the offset - _origfile_stream.seekg(COPY_offset, ios::beg); - // read the copy bytes from original file and write them to output PN_uint32 bytes_left = (PN_uint32)COPY_length; @@ -338,8 +347,8 @@ run(void) { if ((0 == ADD_length) && (0 == COPY_length)) { cleanup(); - if (express_cat.is_spam()) { - express_cat.spam() + if (express_cat.is_debug()) { + express_cat.debug() << "result file = " << _result_file_length << " total bytes = " << _total_bytes_processed << endl; } @@ -349,6 +358,10 @@ run(void) { HashVal MD5_actual; md5_a_file(_temp_file, MD5_actual); if (_MD5_ofResult != MD5_actual) { + // Whoops, patching screwed up somehow. + _origfile_stream.close(); + _write_stream.close(); + express_cat.info() << "Patching produced incorrect checksum. Got:\n" << " " << MD5_actual @@ -356,6 +369,29 @@ run(void) { << " " << _MD5_ofResult << "\n"; + // This is a fine time to double-check the starting + // checksum. + if (!has_source_hash()) { + express_cat.info() + << "No source hash in patch file to verify.\n"; + } else { + HashVal MD5_orig; + md5_a_file(_orig_file, MD5_orig); + if (MD5_orig != get_source_hash()) { + express_cat.info() + << "Started from incorrect source file. Got:\n" + << " " << MD5_orig + << "\nExpected:\n" + << " " << get_source_hash() + << "\n"; + } else { + express_cat.info() + << "Started from correct source file:\n" + << " " << MD5_orig + << "\n"; + } + } + // delete the temp file and the patch file if (!keep_temporary_files) { _temp_file.unlink(); @@ -414,6 +450,73 @@ apply(Filename &patch_file, Filename &file) { return false; } +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::internal_read_header +// Access: Private +// Description: Reads the header and leaves the patch file open. +//////////////////////////////////////////////////////////////////// +int Patchfile:: +internal_read_header(const Filename &patch_file) { + // Open the patch file for read + _patch_file = patch_file; + _patch_file.set_binary(); + if (!_patch_file.open_read(_patch_stream)) { + express_cat.error() + << "Patchfile::initiate() - Failed to open file: " << _patch_file << endl; + return get_write_error(); + } + + ///////////// + // read header, make sure the patch file is valid + + StreamReader patch_reader(_patch_stream); + + // check the magic number + nassertr(_buffer->get_length() >= _v0_header_length, false); + PN_uint32 magic_number = patch_reader.get_uint32(); + if (magic_number != _magic_number && magic_number != _v0_magic_number) { + express_cat.error() + << "Invalid patch file: " << _patch_file << endl; + return EU_error_file_invalid; + } + + _version_number = 0; + if (magic_number != _v0_magic_number) { + _version_number = patch_reader.get_uint16(); + } + if (_version_number > _current_version) { + express_cat.error() + << "Can't read version " << _version_number << " patch files: " + << _patch_file << endl; + return EU_error_file_invalid; + } + + if (_version_number >= 1) { + // Get the length of the source file. + _source_file_length = patch_reader.get_uint32(); + + // get the MD5 of the source file. + _MD5_ofSource.set_value(0, patch_reader.get_uint32()); + _MD5_ofSource.set_value(1, patch_reader.get_uint32()); + _MD5_ofSource.set_value(2, patch_reader.get_uint32()); + _MD5_ofSource.set_value(3, patch_reader.get_uint32()); + } + + // get the length of the patched result file + _result_file_length = patch_reader.get_uint32(); + + // get the MD5 of the resultant patched file + _MD5_ofResult.set_value(0, patch_reader.get_uint32()); + _MD5_ofResult.set_value(1, patch_reader.get_uint32()); + _MD5_ofResult.set_value(2, patch_reader.get_uint32()); + _MD5_ofResult.set_value(3, patch_reader.get_uint32()); + + express_cat.debug() + << "Patchfile::initiate() - valid patchfile" << endl; + + return EU_success; +} + //////////////////////////////////////////////////////////////////// ///// PATCH FILE BUILDING MEMBER FUNCTIONS //////////////////////////////////////////////////////////////////// @@ -586,7 +689,7 @@ calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length) { // original file that matches a string in the new file. //////////////////////////////////////////////////////////////////// void Patchfile:: -find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint16 ©_length, +find_longest_match(PN_uint32 new_pos, PN_uint32 ©_pos, PN_uint16 ©_length, PN_uint32 *hash_table, PN_uint32 *link_table, const char* buffer_orig, PN_uint32 length_orig, const char* buffer_new, PN_uint32 length_new) { @@ -600,16 +703,16 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint16 ©_le if (_NULL_VALUE == hash_table[hash_value]) return; - copy_offset = hash_table[hash_value]; + copy_pos = hash_table[hash_value]; // calc match length - copy_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[copy_offset], - min(min((length_new - new_pos),(length_orig - copy_offset)), _MAX_RUN_LENGTH)); + copy_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[copy_pos], + min(min((length_new - new_pos),(length_orig - copy_pos)), _MAX_RUN_LENGTH)); // run through link table, see if we find any longer matches PN_uint32 match_offset; PN_uint16 match_length; - match_offset = link_table[copy_offset]; + match_offset = link_table[copy_pos]; while (match_offset != _NULL_VALUE) { match_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[match_offset], @@ -617,7 +720,7 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint16 ©_le // have we found a longer match? if (match_length > copy_length) { - copy_offset = match_offset; + copy_pos = match_offset; copy_length = match_length; } @@ -632,8 +735,12 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint16 ©_le // Description: //////////////////////////////////////////////////////////////////// void Patchfile:: -emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer) { - //cout << "ADD: " << length << " bytes" << endl; +emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer, + PN_uint32 ADD_pos) { + if (express_cat.is_spam()) { + express_cat.spam() + << "ADD: " << length << " (to " << ADD_pos << ")" << endl; + } // write ADD length StreamWriter patch_writer(write_stream); @@ -651,8 +758,14 @@ emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer) { // Description: //////////////////////////////////////////////////////////////////// void Patchfile:: -emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset) { - //cout << "COPY: " << length << " bytes at offset " << offset << endl; +emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 COPY_pos, + PN_uint32 last_copy_pos, PN_uint32 ADD_pos) { + PN_int32 offset = (int)COPY_pos - (int)last_copy_pos; + if (express_cat.is_spam()) { + express_cat.spam() + << "COPY: " << length << " bytes from offset " << offset + << " (from " << COPY_pos << " to " << ADD_pos << ")" << endl; + } // write COPY length StreamWriter patch_writer(write_stream); @@ -660,7 +773,7 @@ emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset) { if(length > 0) { // write COPY offset - patch_writer.add_uint32(offset); + patch_writer.add_int32(offset); } } @@ -677,8 +790,7 @@ emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset) { // in time. //////////////////////////////////////////////////////////////////// bool Patchfile:: -build(Filename &file_orig, Filename &file_new) { - Filename patch_name; +build(Filename file_orig, Filename file_new, Filename patch_name) { patch_name.set_binary(); START_PROFILE(overall); @@ -705,7 +817,6 @@ build(Filename &file_orig, Filename &file_new) { // Open patch file for write ofstream write_stream; - patch_name = file_orig.get_fullpath() + ".pch"; if (!patch_name.open_write(write_stream)) { express_cat.error() << "Patchfile::build() - Failed to open file: " << patch_name << endl; @@ -714,17 +825,17 @@ build(Filename &file_orig, Filename &file_new) { // read in original file stream_orig.seekg(0, ios::end); - int length_orig = stream_orig.tellg(); - char *buffer_orig = new char[length_orig]; + _source_file_length = stream_orig.tellg(); + char *buffer_orig = new char[_source_file_length]; stream_orig.seekg(0, ios::beg); - stream_orig.read(buffer_orig, length_orig); + stream_orig.read(buffer_orig, _source_file_length); // read in new file stream_new.seekg(0, ios::end); - int length_new = stream_new.tellg(); - char *buffer_new = new char[length_new]; + _result_file_length = stream_new.tellg(); + char *buffer_new = new char[_result_file_length]; stream_new.seekg(0, ios::beg); - stream_new.read(buffer_new, length_new); + stream_new.read(buffer_new, _result_file_length); // close the original and new files (we have em in memory) stream_orig.close(); @@ -736,14 +847,14 @@ build(Filename &file_orig, Filename &file_new) { // allocate hash/link tables PN_uint32* hash_table = new PN_uint32[_HASHTABLESIZE]; - PN_uint32* link_table = new PN_uint32[length_orig]; + PN_uint32* link_table = new PN_uint32[_source_file_length]; END_PROFILE(allocTables, "allocating hash and link tables"); START_PROFILE(buildTables); // build hash and link tables for original file - build_hash_link_tables(buffer_orig, length_orig, hash_table, link_table); + build_hash_link_tables(buffer_orig, _source_file_length, hash_table, link_table); END_PROFILE(buildTables, "building hash and link tables"); @@ -754,16 +865,26 @@ build(Filename &file_orig, Filename &file_new) { // write the patch file header StreamWriter patch_writer(write_stream); patch_writer.add_uint32(_magic_number); - patch_writer.add_uint32(length_new); + patch_writer.add_uint16(_current_version); + patch_writer.add_uint32(_source_file_length); + { + // calc MD5 of original file + md5_a_buffer((const unsigned char*)buffer_orig, (int)_source_file_length, _MD5_ofSource); + // add it to the header + patch_writer.add_uint32(_MD5_ofSource.get_value(0)); + patch_writer.add_uint32(_MD5_ofSource.get_value(1)); + patch_writer.add_uint32(_MD5_ofSource.get_value(2)); + patch_writer.add_uint32(_MD5_ofSource.get_value(3)); + } + patch_writer.add_uint32(_result_file_length); { // calc MD5 of resultant patched file - HashVal md5New; - md5_a_buffer((const unsigned char*)buffer_new, (int)length_new, md5New); + md5_a_buffer((const unsigned char*)buffer_new, (int)_result_file_length, _MD5_ofResult); // add it to the header - patch_writer.add_uint32(md5New.get_value(0)); - patch_writer.add_uint32(md5New.get_value(1)); - patch_writer.add_uint32(md5New.get_value(2)); - patch_writer.add_uint32(md5New.get_value(3)); + patch_writer.add_uint32(_MD5_ofResult.get_value(0)); + patch_writer.add_uint32(_MD5_ofResult.get_value(1)); + patch_writer.add_uint32(_MD5_ofResult.get_value(2)); + patch_writer.add_uint32(_MD5_ofResult.get_value(3)); } END_PROFILE(writeHeader, "writing patch file header"); @@ -772,18 +893,20 @@ build(Filename &file_orig, Filename &file_new) { START_PROFILE(buildPatchfile); PN_uint32 new_pos = 0; - PN_uint32 ADD_offset = new_pos; // this is the offset for the start of ADD operations + PN_uint32 ADD_pos = new_pos; // this is the position for the start of ADD operations - if(((PN_uint32) length_new) >= _footprint_length) + PN_uint32 last_copy_pos = 0; + + if(((PN_uint32) _result_file_length) >= _footprint_length) { - while (new_pos < (length_new - _footprint_length)) { + while (new_pos < (_result_file_length - _footprint_length)) { // find best match for current position - PN_uint32 COPY_offset; + PN_uint32 COPY_pos; PN_uint16 COPY_length; - find_longest_match(new_pos, COPY_offset, COPY_length, hash_table, link_table, - buffer_orig, length_orig, buffer_new, length_new); + find_longest_match(new_pos, COPY_pos, COPY_length, hash_table, link_table, + buffer_orig, _source_file_length, buffer_new, _result_file_length); // if no match or match not longer than footprint length, skip to next byte if (COPY_length < _footprint_length) { @@ -791,32 +914,47 @@ build(Filename &file_orig, Filename &file_new) { new_pos++; } else { // emit ADD for all skipped bytes - emit_ADD(write_stream, new_pos - ADD_offset, &buffer_new[ADD_offset]); + int num_skipped = (int)new_pos - (int)ADD_pos; + while (num_skipped != (PN_uint16)num_skipped) { + // Overflow. This chunk is too large to fit into a single + // ADD block, so we have to write it as multiple ADDs. + static const PN_uint16 max_write = 65535; + emit_ADD(write_stream, max_write, &buffer_new[ADD_pos], ADD_pos); + ADD_pos += max_write; + num_skipped -= max_write; + emit_COPY(write_stream, 0, COPY_pos, last_copy_pos, ADD_pos); + } + + emit_ADD(write_stream, num_skipped, &buffer_new[ADD_pos], ADD_pos); + ADD_pos += num_skipped; + nassertr(ADD_pos == new_pos, false); // emit COPY for matching string - emit_COPY(write_stream, COPY_length, COPY_offset); + emit_COPY(write_stream, COPY_length, COPY_pos, last_copy_pos, ADD_pos); + last_copy_pos = COPY_pos + COPY_length; // skip past match in new_file new_pos += (PN_uint32)COPY_length; - ADD_offset = new_pos; + ADD_pos = new_pos; } } } // are there still more bytes left in the new file? - if ((int)ADD_offset != length_new) { + if ((int)ADD_pos != _result_file_length) { // emit ADD for all remaining bytes - emit_ADD(write_stream, length_new - ADD_offset, &buffer_new[ADD_offset]); + emit_ADD(write_stream, _result_file_length - ADD_pos, &buffer_new[ADD_pos], + ADD_pos); // write null COPY - emit_COPY(write_stream, 0, 0); + emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, _result_file_length); } END_PROFILE(buildPatchfile, "building patch file"); // write terminator (null ADD, null COPY) - emit_ADD(write_stream, 0, NULL); - emit_COPY(write_stream, 0, 0); + emit_ADD(write_stream, 0, NULL, _result_file_length); + emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, _result_file_length); END_PROFILE(overall, "total patch building operation"); diff --git a/panda/src/express/patchfile.h b/panda/src/express/patchfile.h index 44d03a1918..757a86daf1 100644 --- a/panda/src/express/patchfile.h +++ b/panda/src/express/patchfile.h @@ -57,9 +57,10 @@ PUBLISHED: Patchfile(PT(Buffer) buffer); ~Patchfile(void); - bool build(Filename &file_orig, Filename &file_new); + bool build(Filename file_orig, Filename file_new, Filename patch_name); + int read_header(const Filename &patch_file); - int initiate(Filename &patch_file, Filename &file); + int initiate(const Filename &patch_file, const Filename &file); int run(void); bool apply(Filename &patch_file, Filename &file); @@ -67,10 +68,15 @@ PUBLISHED: INLINE float get_progress(void) const; INLINE void set_footprint_length(int length); - INLINE int get_footprint_length(); + INLINE int get_footprint_length(); INLINE void reset_footprint_length(); + INLINE bool has_source_hash() const; + INLINE const HashVal &get_source_hash() const; + INLINE const HashVal &get_result_hash() const; + private: + int internal_read_header(const Filename &patch_file); void init(PT(Buffer) buffer); void cleanup(void); @@ -79,13 +85,16 @@ private: void build_hash_link_tables(const char *buffer_orig, PN_uint32 length_orig, PN_uint32 *hash_table, PN_uint32 *link_table); PN_uint16 calc_hash(const char *buffer); - void find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint16 ©_length, + void find_longest_match(PN_uint32 new_pos, PN_uint32 ©_pos, PN_uint16 ©_length, PN_uint32 *hash_table, PN_uint32 *link_table, const char* buffer_orig, PN_uint32 length_orig, const char* buffer_new, PN_uint32 length_new); PN_uint32 calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length); - void emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer); - void emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset); + void emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer, + PN_uint32 ADD_pos); + void emit_COPY(ofstream &write_stream, PN_uint16 length, + PN_uint32 COPY_pos, PN_uint32 last_copy_pos, + PN_uint32 ADD_pos); static const PN_uint32 _HASHTABLESIZE; static const PN_uint32 _DEFAULT_FOOTPRINT_LENGTH; @@ -100,8 +109,12 @@ protected: // async patch apply state variables bool _initiated; - HashVal _MD5_ofResult; + PN_uint16 _version_number; + HashVal _MD5_ofSource; + PN_uint32 _source_file_length; + + HashVal _MD5_ofResult; PN_uint32 _result_file_length; int _total_bytes_processed; @@ -113,7 +126,9 @@ protected: Filename _orig_file; Filename _temp_file; + static const PN_uint32 _v0_magic_number; static const PN_uint32 _magic_number; + static const PN_uint16 _current_version; }; #include "patchfile.I"