diff --git a/panda/src/downloader/Sources.pp b/panda/src/downloader/Sources.pp index 2c596fec03..994f002511 100644 --- a/panda/src/downloader/Sources.pp +++ b/panda/src/downloader/Sources.pp @@ -1,31 +1,26 @@ #define LOCAL_LIBS event ipc express pandabase #define OTHER_LIBS interrogatedb:c dconfig:c dtoolutil:c dtoolbase:c dtool:m #define USE_ZLIB yes -#define USE_CRYPTO yes #define USE_IPC yes #begin lib_target #define TARGET downloader - + #define SOURCES \ config_downloader.cxx \ config_downloader.h downloadDb.I \ asyncUtility.I asyncUtility.cxx asyncUtility.h \ downloadDb.cxx downloadDb.h \ downloader.I downloader.cxx downloader.h extractor.cxx extractor.h \ - hashVal.cxx hashVal.I hashVal.h \ multiplexStream.I multiplexStream.cxx multiplexStream.h \ multiplexStreamBuf.I multiplexStreamBuf.cxx multiplexStreamBuf.h \ patcher.cxx \ - patcher.h + patcher.h patcher.I #define IF_ZLIB_SOURCES \ decompressor.cxx decompressor.h zcompressor.I zcompressor.cxx \ zcompressor.h download_utils.cxx download_utils.h - #define IF_CRYPTO_SOURCES \ - crypto_utils.cxx crypto_utils.h - #define INSTALL_HEADERS \ asyncUtility.h asyncUtility.I \ config_downloader.h \ @@ -33,10 +28,9 @@ download_utils.h downloadDb.h downloadDb.I \ downloader.h downloader.I \ extractor.h \ - hashVal.I hashVal.h \ multiplexStream.I multiplexStream.h \ multiplexStreamBuf.I multiplexStreamBuf.I \ - patcher.h \ + patcher.h patcher.I \ zcompressor.I zcompressor.h #define IGATESCAN all diff --git a/panda/src/downloader/patcher.cxx b/panda/src/downloader/patcher.cxx index 05cc37ebab..be185a9004 100644 --- a/panda/src/downloader/patcher.cxx +++ b/panda/src/downloader/patcher.cxx @@ -38,13 +38,16 @@ Patcher(PT(Buffer) buffer) { //////////////////////////////////////////////////////////////////// // Function: Patcher::Constructor -// Access: Private +// Access: Private // Description: //////////////////////////////////////////////////////////////////// void Patcher:: init(PT(Buffer) buffer) { nassertv(!buffer.is_null()); _buffer = buffer; + + _patchfile = NULL; + _patchfile = new Patchfile(_buffer); } //////////////////////////////////////////////////////////////////// @@ -54,27 +57,25 @@ init(PT(Buffer) buffer) { //////////////////////////////////////////////////////////////////// Patcher:: ~Patcher(void) { + delete _patchfile; } //////////////////////////////////////////////////////////////////// // Function: Patcher::initiate // Access: Public -// Description: +// Description: //////////////////////////////////////////////////////////////////// int Patcher:: initiate(Filename &patch, Filename &infile) { - Patchfile pfile(_buffer); - if (pfile.apply(patch, infile) == true) - return PS_success; - return PS_error; + return _patchfile->initiate(patch, infile); } //////////////////////////////////////////////////////////////////// // Function: Patcher::run // Access: Public -// Description: +// Description: //////////////////////////////////////////////////////////////////// int Patcher:: run(void) { - return PS_success; + return _patchfile->run(); } diff --git a/panda/src/downloader/patcher.h b/panda/src/downloader/patcher.h index 289f912e0b..6e172febc6 100644 --- a/panda/src/downloader/patcher.h +++ b/panda/src/downloader/patcher.h @@ -15,16 +15,11 @@ #include //////////////////////////////////////////////////////////////////// -// Class : Patcher -// Description : Applys a patch synchronously +// Class : Patcher +// Description : Applies a patch synchronously //////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS Patcher { PUBLISHED: - enum PatcherStatus { - PS_success = 1, - PS_error = -1, - }; - Patcher(void); Patcher(PT(Buffer) buffer); virtual ~Patcher(void); @@ -32,9 +27,14 @@ PUBLISHED: int initiate(Filename &patch, Filename &infile); int run(void); + INLINE float get_progress(void) const; + private: void init(PT(Buffer) buffer); PT(Buffer) _buffer; + Patchfile *_patchfile; }; +#include "patcher.I" + #endif diff --git a/panda/src/express/Sources.pp b/panda/src/express/Sources.pp index b8c6bc0b36..1bac9ee3ef 100644 --- a/panda/src/express/Sources.pp +++ b/panda/src/express/Sources.pp @@ -4,7 +4,8 @@ #begin lib_target #define TARGET express #define USE_NSPR yes - + #define USE_CRYPTO yes + #define SOURCES \ bigEndian.I bigEndian.cxx bigEndian.h buffer.I buffer.cxx buffer.h \ circBuffer.I circBuffer.h clockObject.I clockObject.cxx \ @@ -15,17 +16,22 @@ datagramOutputFile.I datagramOutputFile.h datagramOutputFile.cxx \ datagramSink.I datagramSink.cxx datagramSink.h \ get_config_path.cxx get_config_path.h \ + hashVal.I hashVal.cxx hashVal.h \ indent.I indent.cxx indent.h littleEndian.I \ littleEndian.cxx littleEndian.h memoryUsage.I memoryUsage.cxx \ memoryUsage.h memoryUsagePointers.I memoryUsagePointers.cxx \ memoryUsagePointers.h multifile.I multifile.cxx multifile.h \ - namable.I namable.cxx namable.h numeric_types.h patchfile.I \ - patchfile.cxx patchfile.h pointerTo.I pointerTo.h referenceCount.I \ + namable.I namable.cxx namable.h numeric_types.h \ + pointerTo.I pointerTo.h referenceCount.I \ referenceCount.cxx referenceCount.h tokenBoard.I tokenBoard.h \ trueClock.I trueClock.cxx trueClock.h typeHandle.I typeHandle.cxx \ typeHandle.h typedReferenceCount.I typedReferenceCount.cxx \ typedReferenceCount.h typedef.h error_utils.cxx error_utils.h + #define IF_CRYPTO_SOURCES \ + crypto_utils.cxx crypto_utils.h \ + patchfile.I patchfile.cxx patchfile.h + #define INSTALL_HEADERS \ bigEndian.I bigEndian.h buffer.I buffer.h circBuffer.I \ circBuffer.h clockObject.I clockObject.h config_express.h \ @@ -33,6 +39,7 @@ datagramIterator.h datagramOutputFile.I datagramOutputFile.h \ datagramSink.I datagramSink.h datagramGenerator.I \ datagramGenerator.h get_config_path.h \ + hashVal.I hashVal.h \ indent.I indent.h littleEndian.I littleEndian.h \ memoryUsage.I memoryUsage.h memoryUsagePointers.I \ memoryUsagePointers.h multifile.I multifile.h \ diff --git a/panda/src/downloader/crypto_utils.cxx b/panda/src/express/crypto_utils.cxx similarity index 95% rename from panda/src/downloader/crypto_utils.cxx rename to panda/src/express/crypto_utils.cxx index a2023c1066..83f2ee9537 100644 --- a/panda/src/downloader/crypto_utils.cxx +++ b/panda/src/express/crypto_utils.cxx @@ -17,7 +17,7 @@ USING_NAMESPACE(CryptoPP); USING_NAMESPACE(std); -static uint +static uint read32(istream& is) { unsigned int ret = 0x0; for (int i=0; i<8; ++i) { @@ -61,7 +61,7 @@ read32(istream& is) { return ret; } -void +void md5_a_file(const Filename &name, HashVal &ret) { ostringstream os; MD5 md5; @@ -78,8 +78,8 @@ md5_a_file(const Filename &name, HashVal &ret) { ret.hv[3] = read32(is); } -void -md5_a_buffer(unsigned char* buf, unsigned long len, HashVal& ret) { +void +md5_a_buffer(const unsigned char* buf, unsigned long len, HashVal& ret) { MD5 md5; HashFilter hash(md5); diff --git a/panda/src/downloader/crypto_utils.h b/panda/src/express/crypto_utils.h similarity index 77% rename from panda/src/downloader/crypto_utils.h rename to panda/src/express/crypto_utils.h index c66f97838c..cd68469dd2 100644 --- a/panda/src/downloader/crypto_utils.h +++ b/panda/src/express/crypto_utils.h @@ -8,14 +8,14 @@ #include #include -#include +#include "typedef.h" class HashVal; BEGIN_PUBLISH EXPCL_PANDAEXPRESS void md5_a_file(const Filename &fname, HashVal &ret); -EXPCL_PANDAEXPRESS void md5_a_buffer(uchar *buf, ulong len, HashVal &ret); +EXPCL_PANDAEXPRESS void md5_a_buffer(const uchar *buf, ulong len, HashVal &ret); END_PUBLISH diff --git a/panda/src/downloader/hashVal.I b/panda/src/express/hashVal.I similarity index 100% rename from panda/src/downloader/hashVal.I rename to panda/src/express/hashVal.I diff --git a/panda/src/downloader/hashVal.cxx b/panda/src/express/hashVal.cxx similarity index 100% rename from panda/src/downloader/hashVal.cxx rename to panda/src/express/hashVal.cxx diff --git a/panda/src/downloader/hashVal.h b/panda/src/express/hashVal.h similarity index 97% rename from panda/src/downloader/hashVal.h rename to panda/src/express/hashVal.h index 55f7405c00..6e66dbad5b 100644 --- a/panda/src/downloader/hashVal.h +++ b/panda/src/express/hashVal.h @@ -7,7 +7,7 @@ #define HASHVAL_H #include -#include +#include "typedef.h" #include //////////////////////////////////////////////////////////////////// diff --git a/panda/src/express/patchfile.I b/panda/src/express/patchfile.I index 103ef25009..7486b14e50 100644 --- a/panda/src/express/patchfile.I +++ b/panda/src/express/patchfile.I @@ -3,18 +3,51 @@ // //////////////////////////////////////////////////////////////////// +//#include "config_downloader.h" + //////////////////////////////////////////////////////////////////// -// Function: Patchfile::write_header -// Access: Private -// Description: Writes the header with platform-independent -// byte ordering +// Function: Patchfile::get_progress +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE float Patchfile:: +get_progress(void) const { + if (false == _initiated) { + express_cat.warning() + << "Patchfile::get_progress() - Patch has not been initiated" << endl; + return 0.0; + } + nassertr(_result_file_length > 0, 0.0); + return ((float)_total_bytes_processed / (float)_result_file_length); +} + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::set_footprint_length +// Access: Public +// Description: //////////////////////////////////////////////////////////////////// INLINE void Patchfile:: -write_header(ofstream &write_stream, const string &name) { - _datagram.clear(); - _datagram.add_uint32(_magic_number); - _datagram.add_int32(name.length()); - _datagram.append_data(name.c_str(), name.length()); - string msg = _datagram.get_message(); - write_stream.write((char *)msg.data(), msg.length()); +set_footprint_length(int length) { + nassertv(_footprint_length > 0); + _footprint_length = length; +} + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::get_footprint_length +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE int Patchfile:: +get_footprint_length() { + return _footprint_length; +} + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::reset_footprint_length +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE void Patchfile:: +reset_footprint_length() { + _footprint_length = _DEFAULT_FOOTPRINT_LENGTH; } diff --git a/panda/src/express/patchfile.cxx b/panda/src/express/patchfile.cxx index 0beec16f32..be639cf597 100644 --- a/panda/src/express/patchfile.cxx +++ b/panda/src/express/patchfile.cxx @@ -18,17 +18,47 @@ //////////////////////////////////////////////////////////////////// #include "patchfile.h" #include "config_express.h" +#include "error_utils.h" #include // for tempnam +// this actually slows things down... +//#define USE_MD5_HASH + +#ifdef USE_MD5_HASH +#include "crypto_utils.h" +#include "hashVal.h" +#endif + +// Patch File Format /////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +// [ HEADER ] +// 4 bytes 0xfeebfaab ("magic number") +// 4 bytes length of resulting patched file +// 4 bytes LFN = length of target file's name +// LFN bytes file name string +// +// [ ADD/COPY pairs; repeated N times ] +// 2 bytes AL = ADD length +// AL bytes bytes to add +// 2 bytes CL = COPY length +// 4 bytes offset of data to copy from original file, if CL != 0 +// +// [ TERMINATOR ] +// 2 bytes zero-length ADD +// 2 bytes zero-length COPY +//////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////// // Defines //////////////////////////////////////////////////////////////////// const PN_uint32 Patchfile::_magic_number = 0xfeebfaab; const PN_uint32 Patchfile::_HASHTABLESIZE = PN_uint32(1) << 16; -const PN_uint32 Patchfile::_footprint_length = 16; +const PN_uint32 Patchfile::_DEFAULT_FOOTPRINT_LENGTH = 9; // this gave the smallest patch file for libpanda.dll when tested, 12/20/2000 const PN_uint32 Patchfile::_NULL_VALUE = PN_uint32(0) - 1; +const PN_uint32 Patchfile::_MAX_RUN_LENGTH = (PN_uint32(1) << 16) - 1; //////////////////////////////////////////////////////////////////// // Function: Patchfile::Constructor @@ -58,25 +88,12 @@ Patchfile(PT(Buffer) buffer) { //////////////////////////////////////////////////////////////////// void Patchfile:: init(PT(Buffer) buffer) { + _initiated = false; nassertv(!buffer.is_null()); _buffer = buffer; - // get all set up with a temp file - char *temp_name = tempnam(NULL, "pf"); - _temp_file_name = temp_name; - _temp_file_name.set_binary(); - delete temp_name; + reset_footprint_length(); - reset(); -} - -//////////////////////////////////////////////////////////////////// -// Function: Patchfile::reset -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -void Patchfile:: -reset(void) { _datagram.clear(); } @@ -87,9 +104,276 @@ reset(void) { //////////////////////////////////////////////////////////////////// Patchfile:: ~Patchfile(void) { - _temp_file_name.unlink(); + if (true == _initiated) + cleanup(); } +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::cleanup +// Access: Private +// Description: +//////////////////////////////////////////////////////////////////// +void Patchfile:: +cleanup(void) { + if (false == _initiated) { + express_cat.error() + << "Patchfile::cleanup() - Patching has not been initiated" + << endl; + return; + } + + // close files + _origfile_stream.close(); + _patch_stream.close(); + _write_stream.close(); + + _initiated = false; +} + +//////////////////////////////////////////////////////////////////// +///// PATCH FILE APPLY MEMBER FUNCTIONS +//////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::initiate +// Access: Public +// Description: Set up to apply the patch to the file (original +// file and patch are destroyed in the process). +//////////////////////////////////////////////////////////////////// +int Patchfile:: +initiate(Filename &patch_file, Filename &file) { + const int _header_length = sizeof(PN_uint32) + sizeof(PN_uint32) + sizeof(PN_int32); + + if (true == _initiated) { + express_cat.error() + << "Patchfile::initiate_apply() - Patching has already been initiated" + << endl; + return EU_error_abort; + } + + // Open the patch file for read + _patch_file = patch_file; + _patch_file.set_binary(); + if (!_patch_file.open_read(_patch_stream)) { + express_cat.error() + << "Patchfile::apply() - Failed to open file: " << _patch_file << endl; + return get_write_error(); + } + + // Open the original file for read + _orig_file = file; + _orig_file.set_binary(); + if (!file.open_read(_origfile_stream)) { + express_cat.error() + << "Patchfile::apply() - Failed to open file: " << file << endl; + return get_write_error(); + } + + // Open the temp file for write + { + char *tempfilename = _tempnam(".", "pf"); + if (NULL == tempfilename) { + express_cat.error() + << "Patchfile::apply() - Failed to create temp file name, using default" << endl; + _temp_file = "patcher_temp_file"; + } else { + //cout << tempfilename << endl; + _temp_file = tempfilename; + free(tempfilename); + } + } + _temp_file.set_binary(); + if (!_temp_file.open_write(_write_stream)) { + express_cat.error() + << "Patchfile::apply() - Failed to open file: " << _temp_file << endl; + return get_write_error(); + } + + ///////////// + // read header, make sure the patch file is valid + + // check the magic number + nassertr(_buffer->get_length() >= _header_length, false); + _patch_stream.read(_buffer->_buffer, _header_length); + _datagram.clear(); + _datagram.append_data(_buffer->_buffer, _header_length); + DatagramIterator di(_datagram); + PN_uint32 magic_number = di.get_uint32(); + if (magic_number != _magic_number) { + express_cat.error() + << "Patchfile::apply() - invalid patch file: " << _patch_file << endl; + //return PF_error_invalidpatchfile; + return EU_error_abort; + } + + // get the length of the patched result file + _result_file_length = di.get_uint32(); + + // check the filename + PN_int32 name_length = di.get_int32(); + nassertr(_buffer->get_length() >= name_length, false); + _patch_stream.read(_buffer->_buffer, name_length); + _datagram.clear(); + _datagram.append_data(_buffer->_buffer, name_length); + DatagramIterator di2(_datagram); + string name = di2.extract_bytes(name_length); + if (name != file.get_basename()) { + express_cat.error() + << "Patchfile::apply() - patch intended for file: " << name + << ", not file: " << file << endl; + //return PF_error_wrongpatchfile; + return EU_error_abort; + } + + express_cat.debug() + << "Patchfile::apply() - valid patchfile for file: " << name << endl; + + _total_bytes_processed = 0; + + _initiated = true; + + return EU_success; +} + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::run +// Access: Public +// Description: Perform one buffer's worth of patching +//////////////////////////////////////////////////////////////////// +int Patchfile:: +run(void) { + // Now patch the file using the given buffer + int buflen; + int bytes_read; + PN_uint16 ADD_length; + PN_uint16 COPY_length; + PN_uint32 COPY_offset; + + if (_initiated == false) { + express_cat.error() + << "Patchfile::run_apply() - Patching has not been initiated" + << endl; + return EU_error_abort; + } + + buflen = _buffer->get_length(); + bytes_read = 0; + + while (bytes_read < buflen) { + /////////// + // read # of ADD bytes + nassertr(_buffer->get_length() >= (int)sizeof(ADD_length), false); + _patch_stream.read(_buffer->_buffer, sizeof(ADD_length)); + _datagram.clear(); + _datagram.append_data(_buffer->_buffer, sizeof(ADD_length)); + DatagramIterator di(_datagram); + ADD_length = di.get_uint16(); + + bytes_read += (int)ADD_length; + _total_bytes_processed += (int)ADD_length; + + // if there are bytes to add, read them from patch file and write them to output + if (0 != ADD_length) { + PN_uint32 bytes_left = (PN_uint32)ADD_length; + + //cout << "ADD: " << ADD_length << endl; + + while (bytes_left > 0) { + PN_uint32 bytes_this_time = (int)min(bytes_left, buflen); + _patch_stream.read(_buffer->_buffer, bytes_this_time); + _write_stream.write(_buffer->_buffer, bytes_this_time); + bytes_left -= bytes_this_time; + } + } + + /////////// + // read # of COPY bytes + nassertr(_buffer->get_length() >= (int)sizeof(COPY_length), false); + _patch_stream.read(_buffer->_buffer, sizeof(COPY_length)); + _datagram.clear(); + _datagram.append_data(_buffer->_buffer, sizeof(COPY_length)); + DatagramIterator di2(_datagram); + COPY_length = di2.get_uint16(); + + bytes_read += (int)COPY_length; + _total_bytes_processed += (int)COPY_length; + + // if there are bytes to copy, read them from original file and write them to output + if (0 != COPY_length) { + // read copy offset + nassertr(_buffer->get_length() >= (int)sizeof(COPY_offset), false); + _patch_stream.read(_buffer->_buffer, sizeof(COPY_offset)); + _datagram.clear(); + _datagram.append_data(_buffer->_buffer, sizeof(COPY_offset)); + DatagramIterator di(_datagram); + COPY_offset = di.get_uint32(); + + //cout << "COPY: " << COPY_length << " bytes at offset " << COPY_offset << endl; + + // seek to the offset + _origfile_stream.seekg(COPY_offset, ios::beg); + + // read the copy bytes from original file and write them to output + PN_uint32 bytes_left = (PN_uint32)COPY_length; + + while (bytes_left > 0) { + PN_uint32 bytes_this_time = ((int)bytes_left < buflen) ? bytes_left : buflen; + _origfile_stream.read(_buffer->_buffer, bytes_this_time); + _write_stream.write(_buffer->_buffer, bytes_this_time); + bytes_left -= bytes_this_time; + } + } + + // if we got a pair of zero-length ADD and COPY blocks, we're done + if ((0 == ADD_length) && (0 == COPY_length)) { + cleanup(); + + //cout << _result_file_length << " " << _total_bytes_processed << endl; + + // delete the patch file and the original file + _patch_file.unlink(); + _orig_file.unlink(); + + // rename the temp file + if (!_temp_file.rename_to(_orig_file)) { + express_cat.error() + << "Patchfile::apply() failed to rename temp file to: " << _orig_file + << endl; + //return PF_error_renamingtempfile; + return EU_error_abort; + } + + return EU_success; + } + } + + return EU_ok; +} + +//////////////////////////////////////////////////////////////////// +// Function: Patchfile::apply +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +bool Patchfile:: +apply(Filename &patch_file, Filename &file) { + int ret = initiate(patch_file, file); + if (ret < 0) + return false; + for (;;) { + ret = run(); + if (ret == EU_success) + return true; + if (ret < 0) + return false; + } + return false; +} + +//////////////////////////////////////////////////////////////////// +///// PATCH FILE BUILDING MEMBER FUNCTIONS +//////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////// // Function: Patchfile::calc_hash // Access: Private @@ -97,6 +381,15 @@ Patchfile:: //////////////////////////////////////////////////////////////////// PN_uint16 Patchfile:: calc_hash(const char *buffer) { +#ifdef USE_MD5_HASH + HashVal hash; + + md5_a_buffer((const unsigned char*)buffer, (int)_footprint_length, hash); + + //cout << PN_uint16(hash.get_value(0)) << " "; + + return PN_uint16(hash.get_value(0)); +#else PN_uint16 hash_value = 0; for(int i = 0; i < (int)_footprint_length; i++) { @@ -105,13 +398,33 @@ calc_hash(const char *buffer) { buffer++; } + //cout << hash_value << " "; + return hash_value; +#endif } //////////////////////////////////////////////////////////////////// // Function: Patchfile::build_hash_link_tables // Access: Private // Description: +// The hash and link tables allow for a quick, linear +// search of all locations in the file that begin with +// a particular sequence of bytes, or "footprint." +// +// The hash table is a table of offsets into the file, +// with one entry for every possible footprint hash +// value. For a hash of a footprint, the entry at the +// offset of the hash value provides an initial location +// in the file that has a matching footprint. +// +// The link table is a large linked list of file offsets, +// with one entry for every byte in the file. Each offset +// in the link table will point to another offset that +// has the same footprint at the corresponding offset in the +// actual file. Starting with an offset taken from the hash +// table, one can rapidly produce a list of offsets that +// all have the same footprint. //////////////////////////////////////////////////////////////////// void Patchfile:: build_hash_link_tables(const char *buffer_orig, PN_uint32 length_orig, @@ -129,6 +442,8 @@ build_hash_link_tables(const char *buffer_orig, PN_uint32 length_orig, link_table[i] = _NULL_VALUE; } + if(length_orig < _footprint_length) return; + // run through original file and hash each footprint for(i = 0; i < (length_orig - _footprint_length); i++) { PN_uint16 hash_value = calc_hash(&buffer_orig[i]); @@ -170,7 +485,7 @@ calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length) { // original file that matches a string in the new file. //////////////////////////////////////////////////////////////////// void Patchfile:: -find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint32 ©_length, +find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint16 ©_length, PN_uint32 *hash_table, PN_uint32 *link_table, const char* buffer_orig, PN_uint32 length_orig, const char* buffer_new, PN_uint32 length_new) { @@ -187,16 +502,17 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint32 ©_le copy_offset = hash_table[hash_value]; // calc match length - copy_length = calc_match_length(&buffer_new[new_pos], &buffer_orig[copy_offset], - min((length_new - new_pos),(length_orig - copy_offset))); + copy_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[copy_offset], + min(min((length_new - new_pos),(length_orig - copy_offset)), _MAX_RUN_LENGTH)); // run through link table, see if we find any longer matches - PN_uint32 match_offset, match_length; + PN_uint32 match_offset; + PN_uint16 match_length; match_offset = link_table[copy_offset]; while (match_offset != _NULL_VALUE) { - match_length = calc_match_length(&buffer_new[new_pos], &buffer_orig[match_offset], - min((length_new - new_pos),(length_orig - match_offset))); + match_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[match_offset], + min(min((length_new - new_pos),(length_orig - match_offset)), _MAX_RUN_LENGTH)); // have we found a longer match? if (match_length > copy_length) { @@ -215,12 +531,12 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint32 ©_le // Description: //////////////////////////////////////////////////////////////////// void Patchfile:: -emit_ADD(ofstream &write_stream, PN_uint32 length, const char* buffer) { -// cout << "ADD: " << length << " bytes" << endl; +emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer) { + //cout << "ADD: " << length << " bytes" << endl; // write ADD length _datagram.clear(); - _datagram.add_uint32(length); + _datagram.add_uint16(length); string msg = _datagram.get_message(); write_stream.write((char *)msg.data(), msg.length()); @@ -239,12 +555,12 @@ emit_ADD(ofstream &write_stream, PN_uint32 length, const char* buffer) { // Description: //////////////////////////////////////////////////////////////////// void Patchfile:: -emit_COPY(ofstream &write_stream, PN_uint32 length, PN_uint32 offset) { -// cout << "COPY: " << length << " bytes at offset " << offset << endl; +emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset) { + //cout << "COPY: " << length << " bytes at offset " << offset << endl; // write COPY length _datagram.clear(); - _datagram.add_uint32(length); + _datagram.add_uint16(length); string msg = _datagram.get_message(); write_stream.write((char *)msg.data(), msg.length()); @@ -326,7 +642,7 @@ build(Filename &file_orig, Filename &file_new) { // build hash and link tables for original file build_hash_link_tables(buffer_orig, length_orig, hash_table, link_table); - // write the patch file header + // prepare to write the patch file header // Strip the v# out of the filename // Save the original extension string ext = file_orig.get_extension(); @@ -334,36 +650,50 @@ build(Filename &file_orig, Filename &file_new) { Filename tfile = file_orig.get_basename_wo_extension(); // Now strip out the .v# string fname = tfile.get_basename_wo_extension(); - fname += "."; - fname += ext; - write_header(write_stream, fname); + if(ext != "") { + fname += "."; + fname += ext; + } + + // write the patch file header + _datagram.clear(); + _datagram.add_uint32(_magic_number); + _datagram.add_uint32(length_new); + _datagram.add_int32(fname.length()); + _datagram.append_data(fname.c_str(), fname.length()); + string msg = _datagram.get_message(); + write_stream.write((char *)msg.data(), msg.length()); // run through new file PN_uint32 new_pos = 0; PN_uint32 ADD_offset = new_pos; // this is the offset for the start of ADD operations - while (new_pos < (length_new - _footprint_length)) { + if(length_new >= _footprint_length) + { + while (new_pos < (length_new - _footprint_length)) { - // find best match for current position - PN_uint32 COPY_offset, COPY_length; + // find best match for current position + PN_uint32 COPY_offset; + PN_uint16 COPY_length; - find_longest_match(new_pos, COPY_offset, COPY_length, hash_table, link_table, - buffer_orig, length_orig, buffer_new, length_new); + find_longest_match(new_pos, COPY_offset, COPY_length, hash_table, link_table, + buffer_orig, length_orig, buffer_new, length_new); - // if no match or match not longer than footprint length, skip to next byte - if (COPY_length < _footprint_length) { - // go to next byte - new_pos++; - } else { - // emit ADD for all skipped bytes - emit_ADD(write_stream, new_pos - ADD_offset, &buffer_new[ADD_offset]); + // if no match or match not longer than footprint length, skip to next byte + if (COPY_length < _footprint_length) { + // go to next byte + new_pos++; + } else { + // emit ADD for all skipped bytes + emit_ADD(write_stream, new_pos - ADD_offset, &buffer_new[ADD_offset]); - // emit COPY for matching string - emit_COPY(write_stream, COPY_length, COPY_offset); + // emit COPY for matching string + emit_COPY(write_stream, COPY_length, COPY_offset); - // skip past match in new_file - new_pos += COPY_length; - ADD_offset = new_pos; + // skip past match in new_file + new_pos += (PN_uint32)COPY_length; + ADD_offset = new_pos; + } } } @@ -383,209 +713,3 @@ build(Filename &file_orig, Filename &file_new) { return true; } -//////////////////////////////////////////////////////////////////// -// Function: Patchfile::apply -// Access: Public -// Description: Apply the patch to the file (original file and -// patch are destroyed in the process). -//////////////////////////////////////////////////////////////////// -bool Patchfile:: -apply(Filename &patch, Filename &file) { - const int _header_length = sizeof(PN_uint32) + sizeof(PN_int32); - - // Open the patch file for read - ifstream patch_stream; - patch.set_binary(); - if (!patch.open_read(patch_stream)) { - express_cat.error() - << "Patchfile::apply() - Failed to open file: " << patch << endl; - return false; - } - - // Open the original file for read - ifstream origfile_stream; - file.set_binary(); - if (!file.open_read(origfile_stream)) { - express_cat.error() - << "Patchfile::apply() - Failed to open file: " << file << endl; - return false; - } - - // Open the temp file for write - ofstream write_stream; - Filename mofile = "patcher_temp_file"; - mofile.set_binary(); - if (!mofile.open_write(write_stream)) { - express_cat.error() - << "Patchfile::apply() - Failed to open file: " << mofile << endl; - return false; - } - - ///////////// - // read header, make sure the patch file is valid - - // check the magic number - nassertr(_buffer->get_length() >= _header_length, false); - patch_stream.read(_buffer->_buffer, _header_length); - _datagram.clear(); - _datagram.append_data(_buffer->_buffer, _header_length); - DatagramIterator di(_datagram); - PN_uint32 magic_number = di.get_uint32(); - if (magic_number != _magic_number) { - express_cat.error() - << "Patchfile::apply() - invalid patch file: " << patch << endl; - return false; - } - - // check the filename - PN_int32 name_length = di.get_int32(); - nassertr(_buffer->get_length() >= name_length, false); - patch_stream.read(_buffer->_buffer, name_length); - _datagram.clear(); - _datagram.append_data(_buffer->_buffer, name_length); - DatagramIterator di2(_datagram); - string name = di2.extract_bytes(name_length); - if (name != file.get_basename()) { - express_cat.error() - << "Patchfile::apply() - patch intended for file: " << name - << ", not file: " << file << endl; - return false; - } - - express_cat.debug() - << "Patchfile::apply() - valid patchfile for file: " << name << endl; - - // Now patch the file using the given buffer - int buflen = _buffer->get_length(); - int done = 0; - PN_uint32 ADD_length; - PN_uint32 COPY_length; - PN_uint32 COPY_offset; - - while (!done) - { - /////////// - // read # of ADD bytes - nassertr(_buffer->get_length() >= (int)sizeof(ADD_length), false); - patch_stream.read(_buffer->_buffer, sizeof(ADD_length)); - _datagram.clear(); - _datagram.append_data(_buffer->_buffer, sizeof(ADD_length)); - DatagramIterator di(_datagram); - ADD_length = di.get_uint32(); - - // if there are bytes to add, read them from patch file and write them to output - if (0 != ADD_length) { - PN_uint32 bytes_left = ADD_length; - - while (bytes_left > 0) { - PN_uint32 bytes_this_time = ((int)bytes_left < buflen) ? bytes_left : buflen; - patch_stream.read(_buffer->_buffer, bytes_this_time); - write_stream.write(_buffer->_buffer, bytes_this_time); - bytes_left -= bytes_this_time; - } - } - - /////////// - // read # of COPY bytes - nassertr(_buffer->get_length() >= (int)sizeof(COPY_length), false); - patch_stream.read(_buffer->_buffer, sizeof(COPY_length)); - _datagram.clear(); - _datagram.append_data(_buffer->_buffer, sizeof(COPY_length)); - DatagramIterator di2(_datagram); - COPY_length = di2.get_uint32(); - - // if there are bytes to copy, read them from original file and write them to output - if (0 != COPY_length) { - // read copy offset - nassertr(_buffer->get_length() >= (int)sizeof(COPY_offset), false); - patch_stream.read(_buffer->_buffer, sizeof(COPY_offset)); - _datagram.clear(); - _datagram.append_data(_buffer->_buffer, sizeof(COPY_offset)); - DatagramIterator di(_datagram); - COPY_offset = di.get_uint32(); - - // seek to the offset - origfile_stream.seekg(COPY_offset, ios::beg); - - // read the copy bytes from original file and write them to output - PN_uint32 bytes_left = COPY_length; - - while (bytes_left > 0) { - PN_uint32 bytes_this_time = ((int)bytes_left < buflen) ? bytes_left : buflen; - origfile_stream.read(_buffer->_buffer, bytes_this_time); - write_stream.write(_buffer->_buffer, bytes_this_time); - bytes_left -= bytes_this_time; - } - } - - // if we got a pair of zero-length ADD and COPY blocks, we're done - if ((0 == ADD_length) && (0 == COPY_length)) { - done = 1; - } - } - - // close files - patch_stream.close(); - origfile_stream.close(); - write_stream.close(); - - // delete the patch file and the original file - patch.unlink(); - file.unlink(); - - // rename the temp file - if (!mofile.rename_to(file)) { - express_cat.error() - << "Patchfile::apply() failed to rename temp file to: " << file - << endl; - return false; - } - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Patchfile::find_longest_sequence -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -int Patchfile:: -find_longest_sequence(Filename &infile, int &pos, int &len) const { - // Open the file for read - ifstream read_stream; - infile.set_binary(); - if (!infile.open_read(read_stream)) { - express_cat.error() - << "Patchfile::find_longest_sequence() - Failed to open file: " - << infile << endl; - return 0; - } - - // Determine file length - read_stream.seekg(0, ios::end); - len = read_stream.tellg(); - char *buffer = new char[len]; - read_stream.seekg(0, ios::beg); - read_stream.read(buffer, len); - - pos = 0; - char holder = 0; - int seq_len; - int longest_seq_len = 0; - for (int i = 0; i < len; i++) { - if (buffer[i] != holder) { - holder = buffer[i]; - seq_len = 0; - } else { - if (++seq_len > longest_seq_len) { - longest_seq_len = seq_len; - pos = i; - } - } - } - - read_stream.close(); - - return longest_seq_len; -} - diff --git a/panda/src/express/patchfile.h b/panda/src/express/patchfile.h index 6ba8302b27..c8af9dfb59 100644 --- a/panda/src/express/patchfile.h +++ b/panda/src/express/patchfile.h @@ -28,41 +28,76 @@ //////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS Patchfile { PUBLISHED: + /* + enum PatchfileStatus { + PF_ok = 2, + PF_success = 1, + PF_error = -1, + PF_error_filenotfound = -2, + PF_error_invalidpatchfile = -3, + PF_error_wrongpatchfile = -4, + PF_error_renamingtempfile = -5, + }; + */ + Patchfile(void); Patchfile(PT(Buffer) buffer); ~Patchfile(void); bool build(Filename &file_orig, Filename &file_new); - bool apply(Filename &patch, Filename &file); - int find_longest_sequence(Filename &infile, int &pos, int &len) const; + int initiate(Filename &patch_file, Filename &file); + int run(void); - void reset(void); + bool apply(Filename &patch_file, Filename &file); + + INLINE float get_progress(void) const; + + INLINE void set_footprint_length(int length); + INLINE int get_footprint_length(); + INLINE void reset_footprint_length(); private: void init(PT(Buffer) buffer); + void cleanup(void); private: // stuff for the build operation - INLINE void write_header(ofstream &write_stream, const string &name); void build_hash_link_tables(const char *buffer_orig, PN_uint32 length_orig, PN_uint32 *hash_table, PN_uint32 *link_table); PN_uint16 calc_hash(const char *buffer); - void find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint32 ©_length, + void find_longest_match(PN_uint32 new_pos, PN_uint32 ©_offset, PN_uint16 ©_length, PN_uint32 *hash_table, PN_uint32 *link_table, const char* buffer_orig, PN_uint32 length_orig, const char* buffer_new, PN_uint32 length_new); PN_uint32 calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length); - void emit_ADD(ofstream &write_stream, PN_uint32 length, const char* buffer); - void emit_COPY(ofstream &write_stream, PN_uint32 length, PN_uint32 offset); + void emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer); + void emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset); static const PN_uint32 _HASHTABLESIZE; - static const PN_uint32 _footprint_length; + static const PN_uint32 _DEFAULT_FOOTPRINT_LENGTH; static const PN_uint32 _NULL_VALUE; + static const PN_uint32 _MAX_RUN_LENGTH; + + PN_uint32 _footprint_length; protected: PT(Buffer) _buffer; // this is the work buffer for apply -- used to prevent virtual memory swapping - Filename _temp_file_name; // this is the temp file + + // async patch apply state variables + bool _initiated; + + PN_uint32 _result_file_length; + int _total_bytes_processed; + + ifstream _patch_stream; + ofstream _write_stream; + ifstream _origfile_stream; + + Filename _patch_file; + Filename _orig_file; + Filename _temp_file; + Datagram _datagram; // used to eliminate endian problems static const PN_uint32 _magic_number;