RomanNum3ral
/
open_toontown_panda3d
mirror of https://github.com/open-toontown/panda3d.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixes for new publish script, patch fixes

This commit is contained in:
David Rose 2002-11-04 21:22:13 +00:00
parent 672f058697
commit bf98a3c9a9
14 changed files with 552 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
panda/src/downloader/downloadDb.cxx
View File

@ -273,8 +273,8 @@ expand_client_multifile(string mfname) {
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::
// Access: Public
// Function: DownloadDb::read_db
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
DownloadDb::Db DownloadDb::
@ -310,8 +310,8 @@ read_db(Filename &file, bool want_server_info) {
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::
// Access: Public
// Function: DownloadDb::read_db
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
DownloadDb::Db DownloadDb::
@ -337,8 +337,8 @@ read_db(Ramfile &file, bool want_server_info) {
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::
// Access: Public
// Function: DownloadDb::write_db
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
bool DownloadDb::
@ -1075,75 +1075,146 @@ output(ostream &out) const {
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::add_version
// Access: Public
// Description: Note: version numbers start at 1
// Access: Published
// Description: Appends a new version of the file onto the end of the
// list, or changes the hash associated with a version
// previously added.
//
// Note: version numbers start at 1
////////////////////////////////////////////////////////////////////
void DownloadDb::
add_version(const Filename &name, HashVal hash, Version version) {
add_version(const Filename &name, const HashVal &hash, int version) {
nassertv(version >= 1);
// Try to find this name in the map
VersionMap::iterator i = _versions.find(name);
VectorHash &vhash = _versions[name];
int size = vhash.size();
// We should not skip over versions as we add them.
nassertv(version <= size+1);
if (version-1 < size) {
// If you are overwriting an old hash value, just rewrite the value
vhash[version-1] = hash;
// If we did not find it, put a new vectorHash at this name_code
if (i == _versions.end()) {
vectorHash v;
nassertv(version == 1);
v.push_back(hash);
_versions[name] = v;
} else {
int size = (*i).second.size();
// Assert that this version is less than or equal to next version in the list
nassertv(version<=size+1);
// If you are overwriting an old hash value, just insert the new value
if (version-1 < size) {
(*i).second[version-1] = hash;
} else {
// add this hash at the end of the vector
(*i).second.push_back(hash);
}
// Otherwise, extend the vector.
vhash.push_back(hash);
}
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::insert_new_version
// Access: Published
// Description: Inserts a new version 1 copy of the file, sliding all
// the other versions up by one.
////////////////////////////////////////////////////////////////////
void DownloadDb::
insert_new_version(const Filename &name, const HashVal &hash) {
VectorHash &vhash = _versions[name];
vhash.insert(vhash.begin(), hash);
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::has_version
// Access: Public
// Access: Published
// Description: Returns true if the indicated file has version
// information, false otherwise. Some files recorded in
// the database may not bother to track versions.
////////////////////////////////////////////////////////////////////
bool DownloadDb::
has_version(const Filename &name) {
has_version(const Filename &name) const {
return (_versions.find(name) != _versions.end());
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::get_num_versions
// Access: Published
// Description: Returns the number of versions stored for the
// indicated file.
////////////////////////////////////////////////////////////////////
int DownloadDb::
get_num_versions(const Filename &name) const {
VersionMap::const_iterator vmi = _versions.find(name);
if (vmi == _versions.end()) {
return 0;
}
return (int)(*vmi).second.size();
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::set_num_versions
// Access: Published
// Description: Reduces the number of versions of a particular file
// stored in the ddb by throwing away all versions
// higher than the indicated index.
////////////////////////////////////////////////////////////////////
void DownloadDb::
set_num_versions(const Filename &name, int num_versions) {
VersionMap::iterator vmi = _versions.find(name);
if (vmi == _versions.end()) {
nassertv(num_versions == 0);
return;
}
VectorHash &vhash = (*vmi).second;
nassertv(num_versions <= (int)vhash.size());
vhash.erase(vhash.begin() + num_versions, vhash.end());
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::get_version
// Access: Public
// Access: Published
// Description: Returns the version number of this particular file,
// determined by looking up the hash generated from the
// file. Returns -1 if the version number cannot be
// determined.
////////////////////////////////////////////////////////////////////
int DownloadDb::
get_version(const Filename &name, HashVal hash) {
get_version(const Filename &name, const HashVal &hash) const {
VersionMap::const_iterator vmi = _versions.find(name);
if (vmi == _versions.end()) {
downloader_cat.debug()
<< "DownloadDb::get_version() - can't find: " << name << endl;
return -1;
}
vectorHash ulvec = (*vmi).second;
vectorHash::iterator i = find(ulvec.begin(), ulvec.end(), hash);
if (i != ulvec.end())
return (i - ulvec.begin() + 1);
const VectorHash &vhash = (*vmi).second;
VectorHash::const_iterator i = find(vhash.begin(), vhash.end(), hash);
if (i != vhash.end())
return (i - vhash.begin() + 1);
downloader_cat.debug()
<< "DownloadDb::get_version() - can't find hash: " << hash << endl;
return -1;
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::get_hash
// Access: Published
// Description: Returns the MD5 hash associated with the indicated
// version of the indicated file.
////////////////////////////////////////////////////////////////////
const HashVal &DownloadDb::
get_hash(const Filename &name, int version) const {
static HashVal bogus_hash;
VersionMap::const_iterator vmi = _versions.find(name);
if (vmi == _versions.end()) {
downloader_cat.error()
<< "DownloadDb::get_hash() - can't find: " << name << endl;
return bogus_hash;
}
const VectorHash &vhash = (*vmi).second;
if (version < 1 || version > (int)vhash.size()) {
downloader_cat.error()
<< "DownloadDb::get_hash() - no version " << version
<< " for " << name << endl;
return bogus_hash;
}
return vhash[version - 1];
}
////////////////////////////////////////////////////////////////////
// Function: DownloadDb::write_version_map
// Access: Protected
@ -1154,7 +1225,7 @@ write_version_map(ofstream &write_stream) {
_master_datagram.clear();
VersionMap::iterator vmi;
vectorHash::iterator i;
VectorHash::iterator i;
string name;
HashVal hash;
@ -1259,18 +1330,14 @@ read_version_map(istream &read_stream) {
////////////////////////////////////////////////////////////////////
void DownloadDb::
output_version_map(ostream &out) const {
out << " Version Map: " << endl;
out << "Version Map: " << endl;
VersionMap::const_iterator vmi;
vectorHash::const_iterator i;
VectorHash::const_iterator i;
for (vmi = _versions.begin(); vmi != _versions.end(); ++vmi) {
out << " Filename: " << (*vmi).first;
out << " " << (*vmi).first << endl;
for (i = (*vmi).second.begin(); i != (*vmi).second.end(); ++i) {
HashVal hash = *i;
out << " [" << hash.get_value(0)
<< " " << hash.get_value(1)
<< " " << hash.get_value(2)
<< " " << hash.get_value(3)
<< "]" << endl;
out << " " << hash << endl;
}
}
out << endl;

20
panda/src/downloader/downloadDb.h
View File

@ -52,7 +52,6 @@ A Db is a Vector<MultifileRecord>
MultifileRecord is a Vector<FileRecord>
*/
typedef int Version;
typedef float Phase;
////////////////////////////////////////////////////////////////////
@ -202,21 +201,26 @@ PUBLISHED:
bool write_db(Filename &file, Db db, bool want_server_info);
public:
// The download db stores two databases, one that represents the client's state
// and one that represents the server state
// The download db stores two databases, one that represents the
// client's state and one that represents the server state.
Db _client_db;
Db _server_db;
// Magic number for knowing this is a download Db
static PN_uint32 _magic_number;
static PN_uint32 _bogus_magic_number;
typedef pvector<HashVal> vectorHash;
typedef pmap<Filename, vectorHash> VersionMap;
typedef pvector<HashVal> VectorHash;
typedef pmap<Filename, VectorHash> VersionMap;
PUBLISHED:
void add_version(const Filename &name, HashVal hash, Version version);
bool has_version(const Filename &name);
int get_version(const Filename &name, HashVal hash);
void add_version(const Filename &name, const HashVal &hash, int version);
void insert_new_version(const Filename &name, const HashVal &hash);
bool has_version(const Filename &name) const;
int get_num_versions(const Filename &name) const;
void set_num_versions(const Filename &name, int num_versions);
int get_version(const Filename &name, const HashVal &hash) const;
const HashVal &get_hash(const Filename &name, int version) const;
protected:
void write_version_map(ofstream &write_stream);

9
panda/src/downloadertools/Sources.pp
View File

@ -22,6 +22,15 @@
#end bin_target
#begin bin_target
#define TARGET show_ddb
#define BUILD_TARGET $[HAVE_CRYPTO]
#define SOURCES \
show_ddb.cxx
#end bin_target
#begin bin_target
#define TARGET check_adler
#define BUILD_TARGET $[HAVE_ZLIB]

17
panda/src/downloadertools/build_patch.cxx
View File

@ -16,14 +16,14 @@
//
////////////////////////////////////////////////////////////////////
#include <pandabase.h>
#include "pandabase.h"
#ifndef HAVE_GETOPT
#include <gnu_getopt.h>
#include "gnu_getopt.h"
#else
#include <getopt.h>
#endif
#include <patchfile.h>
#include <filename.h>
#include "patchfile.h"
#include "filename.h"
int
main(int argc, char *argv[]) {
@ -32,17 +32,18 @@ main(int argc, char *argv[]) {
return 1;
}
Filename src_file = argv[1];
Filename src_file = Filename::from_os_specific(argv[1]);
src_file.set_binary();
Filename dest_file = argv[2];
Filename dest_file = Filename::from_os_specific(argv[2]);
dest_file.set_binary();
Filename patch_file = dest_file.get_fullpath() + ".pch";
Patchfile pfile;
cerr << "Building patch file to convert " << src_file << " to "
<< dest_file << endl;
if (pfile.build(src_file, dest_file) == false) {
<< dest_file << endl;
if (pfile.build(src_file, dest_file, patch_file) == false) {
cerr << "build patch failed" << endl;
return 1;
}

33
panda/src/downloadertools/check_md5.cxx
View File

@ -16,32 +16,20 @@
//
////////////////////////////////////////////////////////////////////
#include <crypto_utils.h>
#include <hashVal.h>
#include <string.h>
#include "crypto_utils.h"
#include "hashVal.h"
#include "filename.h"
int
main(int argc, char *argv[]) {
const char *usagestr="Usage: check_md5 [-dbfmt_output] <file>";
const char *usagestr = "Usage: check_md5 <file>";
if (argc < 2) {
cerr << usagestr << endl;
return 1;
}
bool bRemoveBrackets = (strcmp("-dbfmt_output",argv[1])==0);
Filename source_file;
if(bRemoveBrackets) {
if(argc<3) {
cerr << usagestr << endl;
return 1;
}
source_file = Filename::from_os_specific(argv[2]);
} else {
source_file = Filename::from_os_specific(argv[1]);
}
source_file = Filename::from_os_specific(argv[1]);
if(!source_file.exists()) {
cerr << usagestr << endl;
@ -52,13 +40,10 @@ main(int argc, char *argv[]) {
HashVal hash;
md5_a_file(source_file, hash);
if(bRemoveBrackets) {
hash.set_output_brackets(false);
// output base of Filename along w/md5
cout << source_file.get_basename() << " ";
}
cout << hash << endl;
// output base of Filename along w/md5
cout << source_file.get_basename() << " ";
hash.output(cout);
cout << endl;
return 0;
}

23
panda/src/downloadertools/pcompress.cxx
View File

@ -23,14 +23,20 @@
int
main(int argc, char *argv[]) {
if (argc < 2) {
cerr << "Usage: pcompress <file>" << endl;
cerr << "Usage: pcompress <file> [<dest_file>]" << endl;
return 1;
}
Filename source_file = argv[1];
string dname = argv[1];
dname += ".pz";
Filename dest_file = dname;
bool implicit_dest_file;
Filename source_file = Filename::from_os_specific(argv[1]);
Filename dest_file;
if (argc < 3) {
dest_file = source_file.get_fullpath() + ".pz";
implicit_dest_file = true;
} else {
dest_file = Filename::from_os_specific(argv[2]);
implicit_dest_file = false;
}
// Open source file
ifstream read_stream;
@ -53,7 +59,7 @@ main(int argc, char *argv[]) {
// Open destination file
ofstream write_stream;
dest_file.set_binary();
if (!dest_file.open_write(write_stream)) {
if (!dest_file.open_write(write_stream, true)) {
cerr << "failed to open: " << dest_file << endl;
return 1;
}
@ -70,7 +76,10 @@ main(int argc, char *argv[]) {
read_stream.close();
write_stream.close();
source_file.unlink();
if (implicit_dest_file) {
source_file.unlink();
}
return 0;
}

37
panda/src/downloadertools/show_ddb.cxx Normal file
View File

@ -0,0 +1,37 @@
// Filename: show_ddb.cxx
// Created by: drose (02Nov02)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license. You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////
#include "pandabase.h"
#include "downloadDb.h"
#include "filename.h"
int
main(int argc, char *argv[]) {
if (argc != 3) {
cerr << "Usage: show_ddb server.ddb client.ddb\n";
return 1;
}
Filename server_ddb = Filename::from_os_specific(argv[1]);
Filename client_ddb = Filename::from_os_specific(argv[2]);
DownloadDb db(server_ddb, client_ddb);
db.output_version_map(cout);
return 0;
}

2
panda/src/express/crypto_utils.cxx
View File

@ -79,6 +79,8 @@ read32(istream& is) {
void
md5_a_file(const Filename &name, HashVal &ret) {
nassertv(name.exists());
ostringstream os;
MD5 md5;

42
panda/src/express/hashVal.I
View File

@ -19,18 +19,30 @@
////////////////////////////////////////////////////////////////////
// Function: HashVal::Constructor
// Access: Public
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE HashVal::
HashVal(void) {
HashVal() {
hv[0] = hv[1] = hv[2] = hv[3] = 0;
_bUseBrackets = true;
}
////////////////////////////////////////////////////////////////////
// Function: HashVal::Copy Constructor
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE HashVal::
HashVal(const HashVal &copy) {
hv[0] = copy.hv[0];
hv[1] = copy.hv[1];
hv[2] = copy.hv[2];
hv[3] = copy.hv[3];
}
////////////////////////////////////////////////////////////////////
// Function: HashVal::operator ==
// Access: Public
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE bool HashVal::
@ -43,7 +55,7 @@ operator == (const HashVal &other) const {
////////////////////////////////////////////////////////////////////
// Function: HashVal::operator !=
// Access: Public
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE bool HashVal::
@ -53,7 +65,7 @@ operator != (const HashVal &other) const {
////////////////////////////////////////////////////////////////////
// Function: HashVal::get_value
// Access: Public
// Access: Published
// Description: Returns the integer value of the indicated component.
////////////////////////////////////////////////////////////////////
INLINE uint HashVal::
@ -65,7 +77,7 @@ get_value(int val) const {
////////////////////////////////////////////////////////////////////
// Function: HashVal::set_value
// Access: Public
// Access: Published
// Description: Sets the hash value at index val
////////////////////////////////////////////////////////////////////
INLINE void HashVal::
@ -77,21 +89,11 @@ set_value(int val, uint hashval) {
////////////////////////////////////////////////////////////////////
// Function: HashVal::output
// Access: Public
// Description:
// Access: Published
// Description: The output method does not itself output enclosing
// brackets, but the ostream operator << does.
////////////////////////////////////////////////////////////////////
INLINE void HashVal::
output(ostream &out) const {
if(_bUseBrackets)
out << "[";
out << hv[0] << " " << hv[1] << " " << hv[2] << " " << hv[3];
if(_bUseBrackets)
out << "]";
}
INLINE void HashVal::
set_output_brackets(bool bUseBrackets) {
_bUseBrackets = bUseBrackets;
}

11
panda/src/express/hashVal.cxx
View File

@ -19,3 +19,14 @@
#include "hashVal.h"
////////////////////////////////////////////////////////////////////
// Function: HashVal::as_string
// Access: Published
// Description: Returns the HashVal as a string with four numbers.
////////////////////////////////////////////////////////////////////
string HashVal::
as_string() const {
ostringstream strm;
output(strm);
return strm.str();
}

10
panda/src/express/hashVal.h
View File

@ -30,19 +30,23 @@
class EXPCL_PANDAEXPRESS HashVal {
PUBLISHED:
INLINE HashVal();
INLINE HashVal(const HashVal &copy);
INLINE bool operator == (const HashVal &other) const;
INLINE bool operator != (const HashVal &other) const;
INLINE uint get_value(int val) const;
INLINE void set_value(int val, uint hash);
INLINE void output(ostream &out) const;
uint hv[4];
string as_string() const;
public:
INLINE void set_output_brackets(bool bUseBrackets);
bool _bUseBrackets;
uint hv[4];
};
INLINE ostream &operator << (ostream &out, const HashVal &hv) {
out << "[";
hv.output(out);
out << "]";
return out;
}

42
panda/src/express/patchfile.I
View File

@ -20,7 +20,7 @@
////////////////////////////////////////////////////////////////////
// Function: Patchfile::get_progress
// Access: Public
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE float Patchfile::
@ -36,7 +36,7 @@ get_progress(void) const {
////////////////////////////////////////////////////////////////////
// Function: Patchfile::set_footprint_length
// Access: Public
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE void Patchfile::
@ -47,7 +47,7 @@ set_footprint_length(int length) {
////////////////////////////////////////////////////////////////////
// Function: Patchfile::get_footprint_length
// Access: Public
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE int Patchfile::
@ -57,10 +57,44 @@ get_footprint_length() {
////////////////////////////////////////////////////////////////////
// Function: Patchfile::reset_footprint_length
// Access: Public
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE void Patchfile::
reset_footprint_length() {
_footprint_length = _DEFAULT_FOOTPRINT_LENGTH;
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::has_source_hash
// Access: Published
// Description: Returns true if the MD5 hash for the source file is
// known. (Some early versions of the patch file did
// not store this information.)
////////////////////////////////////////////////////////////////////
INLINE bool Patchfile::
has_source_hash() const {
return (_version_number >= 1);
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::get_source_hash
// Access: Published
// Description: Returns the MD5 hash for the source file.
////////////////////////////////////////////////////////////////////
INLINE const HashVal &Patchfile::
get_source_hash() const {
nassertr(has_source_hash(), _MD5_ofSource);
return _MD5_ofSource;
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::get_result_hash
// Access: Published
// Description: Returns the MD5 hash for the file after the patch has
// been applied.
////////////////////////////////////////////////////////////////////
INLINE const HashVal &Patchfile::
get_result_hash() const {
return _MD5_ofResult;
}

352
panda/src/express/patchfile.cxx
View File

@ -50,16 +50,26 @@ ClockObject *globalClock = ClockObject::get_global_clock();
///// IF THIS CHANGES, UPDATE installerApplyPatch.cxx IN THE INSTALLER
////////////////////////////////////////////////////////////////////
// [ HEADER ]
// 4 bytes 0xfeebfaab ("magic number")
// 4 bytes 0xfeebfaac ("magic number")
// (older patch files have a magic number 0xfeebfaab,
// indicating they are version number 0.)
// 2 bytes version number (if magic number == 0xfeebfaac)
// 4 bytes length of starting file (if version >= 1)
// 16 bytes MD5 of starting file (if version >= 1)
// 4 bytes length of resulting patched file
// 16 bytes MD5 of resultant patched file
const int _header_length = sizeof(PN_uint32) + sizeof(PN_uint32) + (4*sizeof(PN_uint32));
const int _v0_header_length = 4 + 4 + 16;
const int _v1_header_length = 4 + 2 + 4 + 16 + 4 + 16;
//
// [ ADD/COPY pairs; repeated N times ]
// 2 bytes AL = ADD length
// AL bytes bytes to add
// 2 bytes CL = COPY length
// 4 bytes offset of data to copy from original file, if CL != 0
// 4 bytes offset of data to copy from original file, if CL != 0.
// If version >= 2, offset is relative to end of previous
// copy block; if version < 2, offset is relative to
// beginning of file.
//
// [ TERMINATOR ]
// 2 bytes zero-length ADD
@ -70,7 +80,12 @@ const int _header_length = sizeof(PN_uint32) + sizeof(PN_uint32) + (4*sizeof(PN_
////////////////////////////////////////////////////////////////////
// Defines
////////////////////////////////////////////////////////////////////
const PN_uint32 Patchfile::_magic_number = 0xfeebfaab;
const PN_uint32 Patchfile::_v0_magic_number = 0xfeebfaab;
const PN_uint32 Patchfile::_magic_number = 0xfeebfaac;
// Created version 1 on 11/2/02 to store length and MD5 of original file.
// To version 2 on 11/2/02 to store copy offsets as relative.
const PN_uint16 Patchfile::_current_version = 2;
const PN_uint32 Patchfile::_HASHTABLESIZE = PN_uint32(1) << 16;
const PN_uint32 Patchfile::_DEFAULT_FOOTPRINT_LENGTH = 9; // this produced the smallest patch file for libpanda.dll when tested, 12/20/2000
@ -108,6 +123,8 @@ init(PT(Buffer) buffer) {
_initiated = false;
nassertv(!buffer.is_null());
_buffer = buffer;
_version_number = 0;
reset_footprint_length();
}
@ -159,28 +176,19 @@ cleanup(void) {
////////////////////////////////////////////////////////////////////
// Function: Patchfile::initiate
// Access: Public
// Access: Published
// Description: Set up to apply the patch to the file (original
// file and patch are destroyed in the process).
////////////////////////////////////////////////////////////////////
int Patchfile::
initiate(Filename &patch_file, Filename &file) {
if (true == _initiated) {
initiate(const Filename &patch_file, const Filename &file) {
if (_initiated) {
express_cat.error()
<< "Patchfile::initiate() - Patching has already been initiated"
<< endl;
return EU_error_abort;
}
// Open the patch file for read
_patch_file = patch_file;
_patch_file.set_binary();
if (!_patch_file.open_read(_patch_stream)) {
express_cat.error()
<< "Patchfile::initiate() - Failed to open file: " << _patch_file << endl;
return get_write_error();
}
// Open the original file for read
_orig_file = file;
_orig_file.set_binary();
@ -215,42 +223,39 @@ initiate(Filename &patch_file, Filename &file) {
<< "Using temporary file " << _temp_file << "\n";
}
/////////////
// read header, make sure the patch file is valid
StreamReader patch_reader(_patch_stream);
// check the magic number
nassertr(_buffer->get_length() >= _header_length, false);
PN_uint32 magic_number = patch_reader.get_uint32();
if (magic_number != _magic_number) {
express_cat.error()
<< "Patchfile::initiate() - invalid patch file: " << _patch_file << endl;
return EU_error_file_invalid;
}
// get the length of the patched result file
_result_file_length = patch_reader.get_uint32();
// get the MD5 of the resultant patched file
_MD5_ofResult.set_value(0, patch_reader.get_uint32());
_MD5_ofResult.set_value(1, patch_reader.get_uint32());
_MD5_ofResult.set_value(2, patch_reader.get_uint32());
_MD5_ofResult.set_value(3, patch_reader.get_uint32());
express_cat.debug()
<< "Patchfile::initiate() - valid patchfile" << endl;
int result = internal_read_header(patch_file);
_total_bytes_processed = 0;
_initiated = true;
return result;
}
return EU_success;
////////////////////////////////////////////////////////////////////
// Function: Patchfile::read_header
// Access: Published
// Description: Opens the patch file for reading, and gets the header
// information from the file but does not begin to do
// any real work. This can be used to query the data
// stored in the patch.
////////////////////////////////////////////////////////////////////
int Patchfile::
read_header(const Filename &patch_file) {
if (_initiated) {
express_cat.error()
<< "Patchfile::initiate() - Patching has already been initiated"
<< endl;
return EU_error_abort;
}
int result = internal_read_header(patch_file);
_patch_stream.close();
return result;
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::run
// Access: Public
// Access: Published
// Description: Perform one buffer's worth of patching
////////////////////////////////////////////////////////////////////
int Patchfile::
@ -260,7 +265,7 @@ run(void) {
int bytes_read;
PN_uint16 ADD_length;
PN_uint16 COPY_length;
PN_uint32 COPY_offset;
PN_int32 COPY_offset;
if (_initiated == false) {
express_cat.error()
@ -284,20 +289,18 @@ run(void) {
_total_bytes_processed += (int)ADD_length;
// if there are bytes to add, read them from patch file and write them to output
if (0 != ADD_length) {
PN_uint32 bytes_left = (PN_uint32)ADD_length;
if (express_cat.is_spam()) {
express_cat.spam()
<< "ADD: " << ADD_length << " (to "
<< _write_stream.tellp() << ")" << endl;
}
if (express_cat.is_spam()) {
express_cat.spam()
<< "ADD: " << ADD_length << endl;
}
while (bytes_left > 0) {
PN_uint32 bytes_this_time = (PN_uint32) min(bytes_left, (PN_uint32) buflen);
_patch_stream.read(_buffer->_buffer, bytes_this_time);
_write_stream.write(_buffer->_buffer, bytes_this_time);
bytes_left -= bytes_this_time;
}
PN_uint32 bytes_left = (PN_uint32)ADD_length;
while (bytes_left > 0) {
PN_uint32 bytes_this_time = (PN_uint32) min(bytes_left, (PN_uint32) buflen);
_patch_stream.read(_buffer->_buffer, bytes_this_time);
_write_stream.write(_buffer->_buffer, bytes_this_time);
bytes_left -= bytes_this_time;
}
///////////
@ -312,17 +315,23 @@ run(void) {
if (0 != COPY_length) {
// read copy offset
nassertr(_buffer->get_length() >= (int)sizeof(COPY_offset), false);
COPY_offset = patch_reader.get_uint32();
COPY_offset = patch_reader.get_int32();
// seek to the copy source pos
if (_version_number < 2) {
_origfile_stream.seekg(COPY_offset, ios::beg);
} else {
_origfile_stream.seekg(COPY_offset, ios::cur);
}
if (express_cat.is_spam()) {
express_cat.spam()
<< "COPY: " << COPY_length << " bytes at offset " << COPY_offset
<< "COPY: " << COPY_length << " bytes from offset "
<< COPY_offset << " (from " << _origfile_stream.tellg()
<< " to " << _write_stream.tellp() << ")"
<< endl;
}
// seek to the offset
_origfile_stream.seekg(COPY_offset, ios::beg);
// read the copy bytes from original file and write them to output
PN_uint32 bytes_left = (PN_uint32)COPY_length;
@ -338,8 +347,8 @@ run(void) {
if ((0 == ADD_length) && (0 == COPY_length)) {
cleanup();
if (express_cat.is_spam()) {
express_cat.spam()
if (express_cat.is_debug()) {
express_cat.debug()
<< "result file = " << _result_file_length
<< " total bytes = " << _total_bytes_processed << endl;
}
@ -349,6 +358,10 @@ run(void) {
HashVal MD5_actual;
md5_a_file(_temp_file, MD5_actual);
if (_MD5_ofResult != MD5_actual) {
// Whoops, patching screwed up somehow.
_origfile_stream.close();
_write_stream.close();
express_cat.info()
<< "Patching produced incorrect checksum. Got:\n"
<< " " << MD5_actual
@ -356,6 +369,29 @@ run(void) {
<< " " << _MD5_ofResult
<< "\n";
// This is a fine time to double-check the starting
// checksum.
if (!has_source_hash()) {
express_cat.info()
<< "No source hash in patch file to verify.\n";
} else {
HashVal MD5_orig;
md5_a_file(_orig_file, MD5_orig);
if (MD5_orig != get_source_hash()) {
express_cat.info()
<< "Started from incorrect source file. Got:\n"
<< " " << MD5_orig
<< "\nExpected:\n"
<< " " << get_source_hash()
<< "\n";
} else {
express_cat.info()
<< "Started from correct source file:\n"
<< " " << MD5_orig
<< "\n";
}
}
// delete the temp file and the patch file
if (!keep_temporary_files) {
_temp_file.unlink();
@ -414,6 +450,73 @@ apply(Filename &patch_file, Filename &file) {
return false;
}
////////////////////////////////////////////////////////////////////
// Function: Patchfile::internal_read_header
// Access: Private
// Description: Reads the header and leaves the patch file open.
////////////////////////////////////////////////////////////////////
int Patchfile::
internal_read_header(const Filename &patch_file) {
// Open the patch file for read
_patch_file = patch_file;
_patch_file.set_binary();
if (!_patch_file.open_read(_patch_stream)) {
express_cat.error()
<< "Patchfile::initiate() - Failed to open file: " << _patch_file << endl;
return get_write_error();
}
/////////////
// read header, make sure the patch file is valid
StreamReader patch_reader(_patch_stream);
// check the magic number
nassertr(_buffer->get_length() >= _v0_header_length, false);
PN_uint32 magic_number = patch_reader.get_uint32();
if (magic_number != _magic_number && magic_number != _v0_magic_number) {
express_cat.error()
<< "Invalid patch file: " << _patch_file << endl;
return EU_error_file_invalid;
}
_version_number = 0;
if (magic_number != _v0_magic_number) {
_version_number = patch_reader.get_uint16();
}
if (_version_number > _current_version) {
express_cat.error()
<< "Can't read version " << _version_number << " patch files: "
<< _patch_file << endl;
return EU_error_file_invalid;
}
if (_version_number >= 1) {
// Get the length of the source file.
_source_file_length = patch_reader.get_uint32();
// get the MD5 of the source file.
_MD5_ofSource.set_value(0, patch_reader.get_uint32());
_MD5_ofSource.set_value(1, patch_reader.get_uint32());
_MD5_ofSource.set_value(2, patch_reader.get_uint32());
_MD5_ofSource.set_value(3, patch_reader.get_uint32());
}
// get the length of the patched result file
_result_file_length = patch_reader.get_uint32();
// get the MD5 of the resultant patched file
_MD5_ofResult.set_value(0, patch_reader.get_uint32());
_MD5_ofResult.set_value(1, patch_reader.get_uint32());
_MD5_ofResult.set_value(2, patch_reader.get_uint32());
_MD5_ofResult.set_value(3, patch_reader.get_uint32());
express_cat.debug()
<< "Patchfile::initiate() - valid patchfile" << endl;
return EU_success;
}
////////////////////////////////////////////////////////////////////
///// PATCH FILE BUILDING MEMBER FUNCTIONS
////////////////////////////////////////////////////////////////////
@ -586,7 +689,7 @@ calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length) {
// original file that matches a string in the new file.
////////////////////////////////////////////////////////////////////
void Patchfile::
find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_offset, PN_uint16 &copy_length,
find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_pos, PN_uint16 &copy_length,
PN_uint32 *hash_table, PN_uint32 *link_table, const char* buffer_orig,
PN_uint32 length_orig, const char* buffer_new, PN_uint32 length_new) {
@ -600,16 +703,16 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_offset, PN_uint16 &copy_le
if (_NULL_VALUE == hash_table[hash_value])
return;
copy_offset = hash_table[hash_value];
copy_pos = hash_table[hash_value];
// calc match length
copy_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[copy_offset],
min(min((length_new - new_pos),(length_orig - copy_offset)), _MAX_RUN_LENGTH));
copy_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[copy_pos],
min(min((length_new - new_pos),(length_orig - copy_pos)), _MAX_RUN_LENGTH));
// run through link table, see if we find any longer matches
PN_uint32 match_offset;
PN_uint16 match_length;
match_offset = link_table[copy_offset];
match_offset = link_table[copy_pos];
while (match_offset != _NULL_VALUE) {
match_length = (PN_uint16)calc_match_length(&buffer_new[new_pos], &buffer_orig[match_offset],
@ -617,7 +720,7 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_offset, PN_uint16 &copy_le
// have we found a longer match?
if (match_length > copy_length) {
copy_offset = match_offset;
copy_pos = match_offset;
copy_length = match_length;
}
@ -632,8 +735,12 @@ find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_offset, PN_uint16 &copy_le
// Description:
////////////////////////////////////////////////////////////////////
void Patchfile::
emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer) {
//cout << "ADD: " << length << " bytes" << endl;
emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer,
PN_uint32 ADD_pos) {
if (express_cat.is_spam()) {
express_cat.spam()
<< "ADD: " << length << " (to " << ADD_pos << ")" << endl;
}
// write ADD length
StreamWriter patch_writer(write_stream);
@ -651,8 +758,14 @@ emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer) {
// Description:
////////////////////////////////////////////////////////////////////
void Patchfile::
emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset) {
//cout << "COPY: " << length << " bytes at offset " << offset << endl;
emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 COPY_pos,
PN_uint32 last_copy_pos, PN_uint32 ADD_pos) {
PN_int32 offset = (int)COPY_pos - (int)last_copy_pos;
if (express_cat.is_spam()) {
express_cat.spam()
<< "COPY: " << length << " bytes from offset " << offset
<< " (from " << COPY_pos << " to " << ADD_pos << ")" << endl;
}
// write COPY length
StreamWriter patch_writer(write_stream);
@ -660,7 +773,7 @@ emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset) {
if(length > 0) {
// write COPY offset
patch_writer.add_uint32(offset);
patch_writer.add_int32(offset);
}
}
@ -677,8 +790,7 @@ emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset) {
// in time.
////////////////////////////////////////////////////////////////////
bool Patchfile::
build(Filename &file_orig, Filename &file_new) {
Filename patch_name;
build(Filename file_orig, Filename file_new, Filename patch_name) {
patch_name.set_binary();
START_PROFILE(overall);
@ -705,7 +817,6 @@ build(Filename &file_orig, Filename &file_new) {
// Open patch file for write
ofstream write_stream;
patch_name = file_orig.get_fullpath() + ".pch";
if (!patch_name.open_write(write_stream)) {
express_cat.error()
<< "Patchfile::build() - Failed to open file: " << patch_name << endl;
@ -714,17 +825,17 @@ build(Filename &file_orig, Filename &file_new) {
// read in original file
stream_orig.seekg(0, ios::end);
int length_orig = stream_orig.tellg();
char *buffer_orig = new char[length_orig];
_source_file_length = stream_orig.tellg();
char *buffer_orig = new char[_source_file_length];
stream_orig.seekg(0, ios::beg);
stream_orig.read(buffer_orig, length_orig);
stream_orig.read(buffer_orig, _source_file_length);
// read in new file
stream_new.seekg(0, ios::end);
int length_new = stream_new.tellg();
char *buffer_new = new char[length_new];
_result_file_length = stream_new.tellg();
char *buffer_new = new char[_result_file_length];
stream_new.seekg(0, ios::beg);
stream_new.read(buffer_new, length_new);
stream_new.read(buffer_new, _result_file_length);
// close the original and new files (we have em in memory)
stream_orig.close();
@ -736,14 +847,14 @@ build(Filename &file_orig, Filename &file_new) {
// allocate hash/link tables
PN_uint32* hash_table = new PN_uint32[_HASHTABLESIZE];
PN_uint32* link_table = new PN_uint32[length_orig];
PN_uint32* link_table = new PN_uint32[_source_file_length];
END_PROFILE(allocTables, "allocating hash and link tables");
START_PROFILE(buildTables);
// build hash and link tables for original file
build_hash_link_tables(buffer_orig, length_orig, hash_table, link_table);
build_hash_link_tables(buffer_orig, _source_file_length, hash_table, link_table);
END_PROFILE(buildTables, "building hash and link tables");
@ -754,16 +865,26 @@ build(Filename &file_orig, Filename &file_new) {
// write the patch file header
StreamWriter patch_writer(write_stream);
patch_writer.add_uint32(_magic_number);
patch_writer.add_uint32(length_new);
patch_writer.add_uint16(_current_version);
patch_writer.add_uint32(_source_file_length);
{
// calc MD5 of original file
md5_a_buffer((const unsigned char*)buffer_orig, (int)_source_file_length, _MD5_ofSource);
// add it to the header
patch_writer.add_uint32(_MD5_ofSource.get_value(0));
patch_writer.add_uint32(_MD5_ofSource.get_value(1));
patch_writer.add_uint32(_MD5_ofSource.get_value(2));
patch_writer.add_uint32(_MD5_ofSource.get_value(3));
}
patch_writer.add_uint32(_result_file_length);
{
// calc MD5 of resultant patched file
HashVal md5New;
md5_a_buffer((const unsigned char*)buffer_new, (int)length_new, md5New);
md5_a_buffer((const unsigned char*)buffer_new, (int)_result_file_length, _MD5_ofResult);
// add it to the header
patch_writer.add_uint32(md5New.get_value(0));
patch_writer.add_uint32(md5New.get_value(1));
patch_writer.add_uint32(md5New.get_value(2));
patch_writer.add_uint32(md5New.get_value(3));
patch_writer.add_uint32(_MD5_ofResult.get_value(0));
patch_writer.add_uint32(_MD5_ofResult.get_value(1));
patch_writer.add_uint32(_MD5_ofResult.get_value(2));
patch_writer.add_uint32(_MD5_ofResult.get_value(3));
}
END_PROFILE(writeHeader, "writing patch file header");
@ -772,18 +893,20 @@ build(Filename &file_orig, Filename &file_new) {
START_PROFILE(buildPatchfile);
PN_uint32 new_pos = 0;
PN_uint32 ADD_offset = new_pos; // this is the offset for the start of ADD operations
PN_uint32 ADD_pos = new_pos; // this is the position for the start of ADD operations
if(((PN_uint32) length_new) >= _footprint_length)
PN_uint32 last_copy_pos = 0;
if(((PN_uint32) _result_file_length) >= _footprint_length)
{
while (new_pos < (length_new - _footprint_length)) {
while (new_pos < (_result_file_length - _footprint_length)) {
// find best match for current position
PN_uint32 COPY_offset;
PN_uint32 COPY_pos;
PN_uint16 COPY_length;
find_longest_match(new_pos, COPY_offset, COPY_length, hash_table, link_table,
buffer_orig, length_orig, buffer_new, length_new);
find_longest_match(new_pos, COPY_pos, COPY_length, hash_table, link_table,
buffer_orig, _source_file_length, buffer_new, _result_file_length);
// if no match or match not longer than footprint length, skip to next byte
if (COPY_length < _footprint_length) {
@ -791,32 +914,47 @@ build(Filename &file_orig, Filename &file_new) {
new_pos++;
} else {
// emit ADD for all skipped bytes
emit_ADD(write_stream, new_pos - ADD_offset, &buffer_new[ADD_offset]);
int num_skipped = (int)new_pos - (int)ADD_pos;
while (num_skipped != (PN_uint16)num_skipped) {
// Overflow. This chunk is too large to fit into a single
// ADD block, so we have to write it as multiple ADDs.
static const PN_uint16 max_write = 65535;
emit_ADD(write_stream, max_write, &buffer_new[ADD_pos], ADD_pos);
ADD_pos += max_write;
num_skipped -= max_write;
emit_COPY(write_stream, 0, COPY_pos, last_copy_pos, ADD_pos);
}
emit_ADD(write_stream, num_skipped, &buffer_new[ADD_pos], ADD_pos);
ADD_pos += num_skipped;
nassertr(ADD_pos == new_pos, false);
// emit COPY for matching string
emit_COPY(write_stream, COPY_length, COPY_offset);
emit_COPY(write_stream, COPY_length, COPY_pos, last_copy_pos, ADD_pos);
last_copy_pos = COPY_pos + COPY_length;
// skip past match in new_file
new_pos += (PN_uint32)COPY_length;
ADD_offset = new_pos;
ADD_pos = new_pos;
}
}
}
// are there still more bytes left in the new file?
if ((int)ADD_offset != length_new) {
if ((int)ADD_pos != _result_file_length) {
// emit ADD for all remaining bytes
emit_ADD(write_stream, length_new - ADD_offset, &buffer_new[ADD_offset]);
emit_ADD(write_stream, _result_file_length - ADD_pos, &buffer_new[ADD_pos],
ADD_pos);
// write null COPY
emit_COPY(write_stream, 0, 0);
emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, _result_file_length);
}
END_PROFILE(buildPatchfile, "building patch file");
// write terminator (null ADD, null COPY)
emit_ADD(write_stream, 0, NULL);
emit_COPY(write_stream, 0, 0);
emit_ADD(write_stream, 0, NULL, _result_file_length);
emit_COPY(write_stream, 0, last_copy_pos, last_copy_pos, _result_file_length);
END_PROFILE(overall, "total patch building operation");

29
panda/src/express/patchfile.h
View File

@ -57,9 +57,10 @@ PUBLISHED:
Patchfile(PT(Buffer) buffer);
~Patchfile(void);
bool build(Filename &file_orig, Filename &file_new);
bool build(Filename file_orig, Filename file_new, Filename patch_name);
int read_header(const Filename &patch_file);
int initiate(Filename &patch_file, Filename &file);
int initiate(const Filename &patch_file, const Filename &file);
int run(void);
bool apply(Filename &patch_file, Filename &file);
@ -67,10 +68,15 @@ PUBLISHED:
INLINE float get_progress(void) const;
INLINE void set_footprint_length(int length);
INLINE int get_footprint_length();
INLINE int get_footprint_length();
INLINE void reset_footprint_length();
INLINE bool has_source_hash() const;
INLINE const HashVal &get_source_hash() const;
INLINE const HashVal &get_result_hash() const;
private:
int internal_read_header(const Filename &patch_file);
void init(PT(Buffer) buffer);
void cleanup(void);
@ -79,13 +85,16 @@ private:
void build_hash_link_tables(const char *buffer_orig, PN_uint32 length_orig,
PN_uint32 *hash_table, PN_uint32 *link_table);
PN_uint16 calc_hash(const char *buffer);
void find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_offset, PN_uint16 &copy_length,
void find_longest_match(PN_uint32 new_pos, PN_uint32 &copy_pos, PN_uint16 &copy_length,
PN_uint32 *hash_table, PN_uint32 *link_table, const char* buffer_orig,
PN_uint32 length_orig, const char* buffer_new, PN_uint32 length_new);
PN_uint32 calc_match_length(const char* buf1, const char* buf2, PN_uint32 max_length);
void emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer);
void emit_COPY(ofstream &write_stream, PN_uint16 length, PN_uint32 offset);
void emit_ADD(ofstream &write_stream, PN_uint16 length, const char* buffer,
PN_uint32 ADD_pos);
void emit_COPY(ofstream &write_stream, PN_uint16 length,
PN_uint32 COPY_pos, PN_uint32 last_copy_pos,
PN_uint32 ADD_pos);
static const PN_uint32 _HASHTABLESIZE;
static const PN_uint32 _DEFAULT_FOOTPRINT_LENGTH;
@ -100,8 +109,12 @@ protected:
// async patch apply state variables
bool _initiated;
HashVal _MD5_ofResult;
PN_uint16 _version_number;
HashVal _MD5_ofSource;
PN_uint32 _source_file_length;
HashVal _MD5_ofResult;
PN_uint32 _result_file_length;
int _total_bytes_processed;
@ -113,7 +126,9 @@ protected:
Filename _orig_file;
Filename _temp_file;
static const PN_uint32 _v0_magic_number;
static const PN_uint32 _magic_number;
static const PN_uint16 _current_version;
};
#include "patchfile.I"