From b1dc8b2fa61259095a093dc5d00f8cf274ea74c3 Mon Sep 17 00:00:00 2001 From: David Rose Date: Fri, 2 Aug 2002 20:59:23 +0000 Subject: [PATCH] new multifile format for upcoming support of direct loading from multifiles --- panda/src/downloader/asyncDecompressor.cxx | 347 ---- panda/src/downloader/asyncDecompressor.h | 64 - panda/src/downloader/asyncDownloader.I | 110 -- panda/src/downloader/asyncDownloader.cxx | 1045 ---------- panda/src/downloader/asyncDownloader.h | 150 -- panda/src/downloader/asyncExtractor.cxx | 262 --- panda/src/downloader/asyncExtractor.h | 58 - panda/src/downloader/extractor.I | 15 +- panda/src/downloader/extractor.cxx | 253 ++- panda/src/downloader/extractor.h | 58 +- panda/src/downloadertools/multify.cxx | 383 +++- panda/src/express/Sources.pp | 114 +- panda/src/express/express_composite2.cxx | 1 + panda/src/{putil => express}/indirectLess.I | 0 panda/src/{putil => express}/indirectLess.h | 2 +- panda/src/express/multifile.I | 187 +- panda/src/express/multifile.cxx | 1673 +++++++++++------ panda/src/express/multifile.h | 191 +- panda/src/{putil => express}/ordered_vector.I | 0 panda/src/{putil => express}/ordered_vector.T | 0 .../src/{putil => express}/ordered_vector.cxx | 0 panda/src/{putil => express}/ordered_vector.h | 1 - .../test_ordered_vector.cxx | 0 panda/src/putil/Sources.pp | 17 +- panda/src/putil/putil_composite2.cxx | 1 - 25 files changed, 1889 insertions(+), 3043 deletions(-) delete mode 100644 panda/src/downloader/asyncDecompressor.cxx delete mode 100644 panda/src/downloader/asyncDecompressor.h delete mode 100644 panda/src/downloader/asyncDownloader.I delete mode 100644 panda/src/downloader/asyncDownloader.cxx delete mode 100644 panda/src/downloader/asyncDownloader.h delete mode 100644 panda/src/downloader/asyncExtractor.cxx delete mode 100644 panda/src/downloader/asyncExtractor.h rename panda/src/{putil => express}/indirectLess.I (100%) rename panda/src/{putil => express}/indirectLess.h (98%) rename panda/src/{putil => express}/ordered_vector.I (100%) rename panda/src/{putil => express}/ordered_vector.T (100%) rename panda/src/{putil => express}/ordered_vector.cxx (100%) rename panda/src/{putil => express}/ordered_vector.h (99%) rename panda/src/{putil => express}/test_ordered_vector.cxx (100%) diff --git a/panda/src/downloader/asyncDecompressor.cxx b/panda/src/downloader/asyncDecompressor.cxx deleted file mode 100644 index de2ed5c42b..0000000000 --- a/panda/src/downloader/asyncDecompressor.cxx +++ /dev/null @@ -1,347 +0,0 @@ -// Filename: asyncDecompressor.cxx -// Created by: mike (09Jan97) -// -//////////////////////////////////////////////////////////////////// -// -// PANDA 3D SOFTWARE -// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved -// -// All use of this software is subject to the terms of the Panda 3d -// Software license. You should have received a copy of this license -// along with this source code; you will also find a current copy of -// the license at http://www.panda3d.org/license.txt . -// -// To contact the maintainers of this program write to -// panda3d@yahoogroups.com . -// -//////////////////////////////////////////////////////////////////// - -// This file is compiled only if we have zlib installed. - -//////////////////////////////////////////////////////////////////// -// Includes -//////////////////////////////////////////////////////////////////// - -#include "config_downloader.h" - -#include -#include -#include -#include -#include -#include - -#include "asyncDecompressor.h" - -//////////////////////////////////////////////////////////////////// -// Defines -//////////////////////////////////////////////////////////////////// - -//////////////////////////////////////////////////////////////////// -// Class : DecompressorToken -// Description : Holds a request for the decompressor. -//////////////////////////////////////////////////////////////////// -class DecompressorToken : public ReferenceCount { -public: - INLINE DecompressorToken(uint id, const Filename &source_file, - const Filename &dest_file, const string &event_name) { - _id = id; - _source_file = source_file; - _dest_file = dest_file; - _event_name = event_name; - } - int _id; - Filename _source_file; - Filename _dest_file; - string _event_name; -}; - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Decompressor:: -Decompressor(void) : AsyncUtility() { - PT(Buffer) buffer = new Buffer(decompressor_buffer_size); - init(buffer); -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Decompressor:: -Decompressor(PT(Buffer) buffer) : AsyncUtility() { - init(buffer); -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::Constructor -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -void Decompressor:: -init(PT(Buffer) buffer) { - nassertv(!buffer.is_null()); - _frequency = decompressor_frequency; - _token_board = new DecompressorTokenBoard; - _half_buffer_length = buffer->get_length()/2; - _buffer = buffer; - char *temp_name = tempnam(NULL, "dc"); - _temp_file_name = temp_name; - _temp_file_name.set_binary(); - delete temp_name; -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::Destructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Decompressor:: -~Decompressor(void) { - destroy_thread(); - - delete _token_board; - _temp_file_name.unlink(); -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::request_decompress -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -int Decompressor:: -request_decompress(const Filename &source_file, const string &event_name) { - Filename dest_file = source_file; - string extension = source_file.get_extension(); - if (extension == "pz") - dest_file = source_file.get_fullpath_wo_extension(); - else { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Decompressor::request_decompress() - Unknown file extension: ." - << extension << endl; - } - return request_decompress(source_file, dest_file, event_name); -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::request_decompress -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -int Decompressor:: -request_decompress(const Filename &source_file, const Filename &dest_file, - const string &event_name) { - - PT(DecompressorToken) tok; - if (_threads_enabled) { - - // Make sure we actually are threaded - if (!_threaded) { - downloader_cat.info() - << "Decompressor::request_decompress() - create_thread() was " - << "never called! Calling it now..." << endl; - create_thread(); - } - - // We need to grab the lock in order to signal the condition variable -#ifdef HAVE_IPC - _lock.lock(); -#endif - - if (_token_board->_waiting.is_full()) { - downloader_cat.error() - << "Downloader::request_download() - Too many pending requests\n"; - return 0; - } - - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Decompress requested for file: " << source_file << endl; - } - - tok = new DecompressorToken(_next_token++, source_file, dest_file, - event_name); - _token_board->_waiting.insert(tok); - -#ifdef HAVE_IPC - _request_cond->signal(); - _lock.unlock(); -#endif - - } else { - // If we're not running asynchronously, process the load request - // directly now. - if (_token_board->_waiting.is_full()) { - downloader_cat.error() - << "Downloader::request_download() - Too many pending requests\n"; - return 0; - } - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Decompress requested for file: " << source_file << endl; - } - - tok = new DecompressorToken(_next_token++, source_file, dest_file, - event_name); - _token_board->_waiting.insert(tok); - process_request(); - } - - return tok->_id; -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::process_request -// Access: Private -// Description: Serves any requests on the token board, moving them -// to the done queue. -//////////////////////////////////////////////////////////////////// -bool Decompressor:: -process_request() { - if (_shutdown) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Decompressor shutting down...\n"; - return false; - } - - // If there is actually a request token - process it - while (!_token_board->_waiting.is_empty()) { - PT(DecompressorToken) tok = _token_board->_waiting.extract(); - if (decompress(tok->_source_file, tok->_dest_file)) { - _token_board->_done.insert(tok); - - // Throw a "done" event now. - if (!tok->_event_name.empty()) { - PT_Event done = new Event(tok->_event_name); - done->add_parameter(EventParameter((int)tok->_id)); - throw_event(done); - } - - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Decompressor::process_request() - decompress complete for " - << tok->_source_file << "\n"; - } - } - } - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::decompress -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -bool Decompressor:: -decompress(Filename &source_file) { - Filename dest_file = source_file; - string extension = source_file.get_extension(); - if (extension == "pz") - dest_file = source_file.get_fullpath_wo_extension(); - else { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Decompressor::request_decompress() - Unknown file extension: ." - << extension << endl; - return false; - } - return decompress(source_file, dest_file); -} - -//////////////////////////////////////////////////////////////////// -// Function: Decompressor::decompress -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -bool Decompressor:: -decompress(Filename &source_file, Filename &dest_file) { - - // Open source file - ifstream read_stream; - source_file.set_binary(); - if (!source_file.open_read(read_stream)) { - downloader_cat.error() - << "Decompressor::decompress() - Error opening source file: " - << source_file << endl; - return false; - } - - // Determine source file length - read_stream.seekg(0, ios::end); - int source_file_length = read_stream.tellg(); - if (source_file_length == 0) { - downloader_cat.warning() - << "Decompressor::decompress() - Zero length file: " - << source_file << endl; - return true; - } - read_stream.seekg(0, ios::beg); - - // Open destination file - ofstream write_stream; - dest_file.set_binary(); - if (!dest_file.open_write(write_stream)) { - downloader_cat.error() - << "Decompressor::decompress() - Error opening dest file: " - << source_file << endl; - return false; - } - - // Read from the source file into the first half of the buffer, - // decompress into the second half of the buffer, write the second - // half of the buffer to disk, and repeat. - int total_bytes_read = 0; - bool read_all_input = false; - bool handled_all_input = false; - int source_buffer_length; - ZDecompressor decompressor; - while (handled_all_input == false) { - - // See if there is anything left in the source file - if (read_all_input == false) { - read_stream.read(_buffer->_buffer, _half_buffer_length); - source_buffer_length = read_stream.gcount(); - total_bytes_read += source_buffer_length; - if (read_stream.eof()) { - nassertr(total_bytes_read == source_file_length, false); - read_all_input = true; - } - } - - char *next_in = _buffer->_buffer; - int avail_in = source_buffer_length; - char *dest_buffer = _buffer->_buffer + source_buffer_length; - char *next_out = dest_buffer; - int dest_buffer_length = _buffer->get_length() - source_buffer_length; - int avail_out = dest_buffer_length; - nassertr(avail_out > 0 && avail_in > 0, false); - - while (avail_in > 0) { - int ret = decompressor.decompress_to_stream(next_in, avail_in, - next_out, avail_out, dest_buffer, - dest_buffer_length, write_stream); - if (ret == ZCompressorBase::S_error) - return false; - if ((int)decompressor.get_total_in() == source_file_length && - avail_out == dest_buffer_length) - handled_all_input = true; - } - - nap(); - - } - - read_stream.close(); - write_stream.close(); - - source_file.unlink(); - - return true; -} diff --git a/panda/src/downloader/asyncDecompressor.h b/panda/src/downloader/asyncDecompressor.h deleted file mode 100644 index 4456bc4646..0000000000 --- a/panda/src/downloader/asyncDecompressor.h +++ /dev/null @@ -1,64 +0,0 @@ -// Filename: asyncDecompressor.h -// Created by: mike (09Jan97) -// -//////////////////////////////////////////////////////////////////// -// -// PANDA 3D SOFTWARE -// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved -// -// All use of this software is subject to the terms of the Panda 3d -// Software license. You should have received a copy of this license -// along with this source code; you will also find a current copy of -// the license at http://www.panda3d.org/license.txt . -// -// To contact the maintainers of this program write to -// panda3d@yahoogroups.com . -// -//////////////////////////////////////////////////////////////////// -#ifndef ASYNCDECOMPRESSOR_H -#define ASYNCDECOMPRESSOR_H -// -//////////////////////////////////////////////////////////////////// -// Includes -//////////////////////////////////////////////////////////////////// -#include -#include -#include -#include -#include "zcompressor.h" -#include "asyncUtility.h" - -class DecompressorToken; - -//////////////////////////////////////////////////////////////////// -// Class : Decompressor -// Description : -//////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS Decompressor : public AsyncUtility { -PUBLISHED: - Decompressor(void); - Decompressor(PT(Buffer) buffer); - virtual ~Decompressor(void); - - int request_decompress(const Filename &source_file, - const string &event_name); - int request_decompress(const Filename &source_file, - const Filename &dest_file, - const string &event_name); - - bool decompress(Filename &source_file); - bool decompress(Filename &source_file, Filename &dest_file); - -private: - void init(PT(Buffer) buffer); - virtual bool process_request(void); - - typedef TokenBoard DecompressorTokenBoard; - DecompressorTokenBoard *_token_board; - - PT(Buffer) _buffer; - int _half_buffer_length; - Filename _temp_file_name; -}; - -#endif diff --git a/panda/src/downloader/asyncDownloader.I b/panda/src/downloader/asyncDownloader.I deleted file mode 100644 index a496e930c7..0000000000 --- a/panda/src/downloader/asyncDownloader.I +++ /dev/null @@ -1,110 +0,0 @@ -// Filename: asyncDownloader.I -// Created by: mike (09Jan97) -// -//////////////////////////////////////////////////////////////////// -// -// PANDA 3D SOFTWARE -// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved -// -// All use of this software is subject to the terms of the Panda 3d -// Software license. You should have received a copy of this license -// along with this source code; you will also find a current copy of -// the license at http://www.panda3d.org/license.txt . -// -// To contact the maintainers of this program write to -// panda3d@yahoogroups.com . -// -//////////////////////////////////////////////////////////////////// - -#include "config_downloader.h" - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::set_byte_rate -// Access: Public -// Description: Note: modem speeds are reported in bits, so you -// need to convert! -//////////////////////////////////////////////////////////////////// -INLINE void Downloader:: -set_byte_rate(float bytes) { - nassertv(bytes > 0.0); - if (bytes == _byte_rate) - return; -#ifdef HAVE_IPC - _buffer_lock.lock(); -#endif - - _new_byte_rate = bytes; - -#ifdef HAVE_IPC - _buffer_lock.unlock(); -#endif -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::get_byte_rate -// Access: Public -// Description: Returns byte rate in bytes. -//////////////////////////////////////////////////////////////////// -INLINE float Downloader:: -get_byte_rate(void) const { - return _byte_rate; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::enable_download -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -INLINE void Downloader:: -enable_download(bool val) { - _download_enabled = val; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::is_download_enabled -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -INLINE bool Downloader:: -is_download_enabled(void) const { - return _download_enabled; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::set_disk_write_frequency -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -INLINE void Downloader:: -set_disk_write_frequency(int frequency) { - nassertv(frequency > 0); -#ifdef HAVE_IPC - _buffer_lock.lock(); -#endif - - _new_disk_write_frequency = frequency; - -#ifdef HAVE_IPC - _buffer_lock.unlock(); -#endif -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::get_disk_write_frequency -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -INLINE int Downloader:: -get_disk_write_frequency(void) const { - return _disk_write_frequency; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::get_last_attempt_stalled -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -INLINE bool Downloader:: -get_last_attempt_stalled(void) const { - return _last_attempt_stalled; -} diff --git a/panda/src/downloader/asyncDownloader.cxx b/panda/src/downloader/asyncDownloader.cxx deleted file mode 100644 index d8216c63fd..0000000000 --- a/panda/src/downloader/asyncDownloader.cxx +++ /dev/null @@ -1,1045 +0,0 @@ -// Filename: asyncDownloader.cxx -// Created by: mike (09Jan97) -// -//////////////////////////////////////////////////////////////////// -// -// PANDA 3D SOFTWARE -// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved -// -// All use of this software is subject to the terms of the Panda 3d -// Software license. You should have received a copy of this license -// along with this source code; you will also find a current copy of -// the license at http://www.panda3d.org/license.txt . -// -// To contact the maintainers of this program write to -// panda3d@yahoogroups.com . -// -//////////////////////////////////////////////////////////////////// - -#include "config_downloader.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "asyncDownloader.h" -#include -#include "plist.h" - -#if !defined(WIN32_VC) -// #define errno wsaGetLastError() - #include - #include - #include - #include -#endif - -//////////////////////////////////////////////////////////////////// -// Defines -//////////////////////////////////////////////////////////////////// -enum send_status { - SS_error, - SS_timeout, - SS_success -}; - -enum receive_status { - RS_error, - RS_timeout, - RS_success, - RS_eof -}; - -//////////////////////////////////////////////////////////////////// -// Class : DownloaderToken -// Description : Holds a request for the downloader. -//////////////////////////////////////////////////////////////////// -class DownloaderToken : public ReferenceCount { -public: - INLINE DownloaderToken(uint id, const string &file_name, - const Filename &file_dest, const string &event_name, - int first_byte, int last_byte, int total_bytes, - bool partial_content, bool sync) : _id(id), _first_byte(first_byte), - _last_byte(last_byte), _total_bytes(total_bytes) { - _file_name = file_name; - _event_name = event_name; - _file_dest = file_dest; - _partial_content = partial_content; - _sync = sync; - } - uint _id; - string _file_name; - Filename _file_dest; - string _event_name; - int _first_byte; - int _last_byte; - int _total_bytes; - bool _partial_content; - bool _sync; -}; - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Downloader:: -Downloader(void) : AsyncUtility() { - init(); -} - -#if 0 -//////////////////////////////////////////////////////////////////// -// Function: Downloader::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Downloader:: -Downloader(PT(Buffer) buffer) : AsyncUtility() { - init(buffer); -} -#endif - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::init -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -void Downloader:: -init(void) { - _disk_write_frequency = downloader_disk_write_frequency; - _new_disk_write_frequency = 0; - _byte_rate = downloader_byte_rate; - _new_byte_rate = 0; - _frequency = downloader_frequency; - nassertv(_frequency > 0 && _byte_rate > 0); - _read_size = _byte_rate * _frequency; - _disk_buffer_size = _disk_write_frequency * _read_size; - _buffer = new Buffer(_disk_buffer_size); - _connected = false; - _token_board = new DownloaderTokenBoard; - _download_enabled = true; - _last_attempt_stalled = true; - // We need to flush after every write in case we're interrupted - _dest_stream.setf(ios::unitbuf, 0); - _last_attempt_stalled = false; - _current_attempt_stalled = false; - -#if defined(WIN32) - WSAData mydata; - int answer1 = WSAStartup(0x0101, &mydata); - if(answer1 != 0) { - downloader_cat.error() - << "Downloader::Downloader() - Error initializing TCP stack!" - << endl; - } -#endif -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::Destructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Downloader:: -~Downloader() { - if (_connected) - disconnect_from_server(); - - destroy_thread(); - - delete _token_board; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::connect_to_server -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -bool Downloader:: -connect_to_server(const string &name, uint port) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader connecting to server: " << name - << " on port: " << port << endl; - - _server_name = name; - - _sin.sin_family = PF_INET; - _sin.sin_port = htons(port); - ulong addr = (ulong)inet_addr(name.c_str()); - struct hostent *hp = NULL; - - if (addr == INADDR_NONE) { - hp = gethostbyname(name.c_str()); - if (hp != NULL) - (void)memcpy(&_sin.sin_addr, hp->h_addr, (uint)hp->h_length); - else { - downloader_cat.error() - << "Downloader::connect_to_server() - gethostbyname() failed: " - << strerror(errno) << endl; - return false; - } - } else - (void)memcpy(&_sin.sin_addr, &addr, sizeof(addr)); - - return connect_to_server(); -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::connect_to_server -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -bool Downloader:: -connect_to_server(void) { - if (_connected == true) - return true; - - _socket = 0xffffffff; - _socket = socket(PF_INET, SOCK_STREAM, 0); - if (_socket == (int)0xffffffff) { - downloader_cat.error() - << "Downloader::connect_to_server() - socket failed: " - << strerror(errno) << endl; - return false; - } - - _connected = true; - - if (connect(_socket, (struct sockaddr *)&_sin, sizeof(_sin)) < 0) { - downloader_cat.error() - << "Downloader::connect_to_server() - connect() failed: " - << strerror(errno) << endl; - disconnect_from_server(); - _connected = false; - } - - return _connected; -} - -/////////////////////////////////////////////////////////////////// -// Function: Downloader::disconnect_from_server -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -void Downloader:: -disconnect_from_server(void) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader disconnecting from server..." << endl; - -#if defined(WIN32) - (void)closesocket(_socket); -#else - (void)close(_socket); -#endif - - _connected = false; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::request_sync_download -// Access: Public -// Description: Requests the synchronous download of a complete file. -//////////////////////////////////////////////////////////////////// -int Downloader:: -request_sync_download(const string &file_name, const Filename &file_dest, - const string &event_name) { - return request_download(file_name, file_dest, event_name, true); -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::request_sync_download -// Access: Public -// Description: Requests the synchronous download of a complete file. -//////////////////////////////////////////////////////////////////// -int Downloader:: -request_sync_download(const string &file_name, const Filename &file_dest, - const string &event_name, int first_byte, - int last_byte, int total_bytes, bool partial_content) { - return request_download(file_name, file_dest, event_name, first_byte, - last_byte, total_bytes, partial_content, true); -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::request_download -// Access: Public -// Description: Requests the download of a complete file. -//////////////////////////////////////////////////////////////////// -int Downloader:: -request_download(const string &file_name, const Filename &file_dest, - const string &event_name, bool sync) { - return request_download(file_name, file_dest, event_name, 0, 0, 0, - false, sync); -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::request_download -// Access: Public -// Description: Requests an asynchronous load of a file. The request -// will be queued and served by the asynchronous thread. -// If event_name is nonempty, it is the name of the -// event that will be thrown (with the uint id as its -// single parameter) when the loading is completed later. -// -// The return value is an integer which can be used to -// identify this particular request later to -// fetch_load(), or 0 if there has been an error. -// -// Can be used to request a partial download of a file. -//////////////////////////////////////////////////////////////////// -int Downloader:: -request_download(const string &file_name, const Filename &file_dest, - const string &event_name, int first_byte, - int last_byte, int total_bytes, - bool partial_content, bool sync) { - - nassertr(first_byte <= last_byte && last_byte <= total_bytes, 0); - - PT(DownloaderToken) tok; - if (_threads_enabled) { - - // Make sure we actually are threaded - if (!_threaded) { - downloader_cat.info() - << "Downloader::request_download() - create_thread() was " - << "never called! Calling it now..." << endl; - create_thread(); - } - - // We need to grab the lock in order to signal the condition variable -#ifdef HAVE_IPC - _lock.lock(); -#endif - - if (_token_board->_waiting.full()) { - downloader_cat.error() - << "Downloader::request_download() - Too many pending requests\n"; - return 0; - } - - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Download requested for file: " << file_name << "\n"; - } - - tok = new DownloaderToken(_next_token++, file_name, file_dest, - event_name, first_byte, last_byte, total_bytes, - partial_content, sync); - _token_board->_waiting.push_back(tok); - -#ifdef HAVE_IPC - _request_cond->signal(); - _lock.unlock(); -#endif - - } else { - // If we're not running asynchronously, process the load request - // directly now. - if (_token_board->_waiting.full()) { - downloader_cat.error() - << "Downloader::request_download() - Too many pending requests\n"; - return 0; - } - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Load requested for file: " << file_name << "\n"; - } - - tok = new DownloaderToken(_next_token++, file_name, file_dest, - event_name, first_byte, last_byte, total_bytes, - partial_content, sync); - _token_board->_waiting.push_back(tok); - process_request(); - } - - return tok->_id; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::process_request -// Access: Private -// Description: Serves any requests on the token board, moving them -// to the done queue. -//////////////////////////////////////////////////////////////////// -bool Downloader:: -process_request() { - if (_shutdown) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader shutting down...\n"; - return false; - } - - // If there is actually a request token - process it - while (!_token_board->_waiting.empty()) { - PT(DownloaderToken) tok = _token_board->_waiting.front(); - _token_board->_waiting.pop_front(); - int ret = download(tok->_file_name, tok->_file_dest, tok->_event_name, - tok->_first_byte, tok->_last_byte, tok->_total_bytes, - tok->_partial_content, tok->_sync, tok->_id); - nassertr(tok->_event_name.empty() == false, false); - PT_Event return_event = new Event(tok->_event_name); - return_event->add_parameter(EventParameter((int)tok->_id)); - if (ret == DS_success) { - _token_board->_done.push_back(tok); - return_event->add_parameter(EventParameter(DS_success)); - - // Throw a "done" event now. - if (!tok->_event_name.empty()) { - PT_Event done = new Event(tok->_event_name); - done->add_parameter(EventParameter((int)tok->_id)); - throw_event(done); - } - - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Downloader::process_request() - downloading complete for " - << tok->_file_name << "\n"; - } - } else { - return_event->add_parameter(EventParameter(ret)); - } - throw_event(return_event); - } - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::safe_send -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -int Downloader:: -safe_send(int socket, const char *data, int length, long timeout) { - if (length == 0) { - downloader_cat.error() - << "Downloader::safe_send() - requested 0 length send!" << endl; - return SS_error; - } - int bytes = 0; - struct timeval tv; - tv.tv_sec = timeout; - tv.tv_usec = 0; - fd_set wset; - FD_ZERO(&wset); - while (bytes < length) { - FD_SET(socket, &wset); - int sret = select(socket + 1, NULL, &wset, NULL, &tv); - if (sret == 0) { - downloader_cat.error() - << "Downloader::safe_send() - select timed out after: " - << timeout << " seconds" << endl; - return SS_timeout; - } else if (sret == -1) { - downloader_cat.error() - << "Downloader::safe_send() - error: " << strerror(errno) << endl; - return SS_error; - } - int ret = send(socket, data, length, 0); - if (ret > 0) - bytes += ret; - else { - downloader_cat.error() - << "Downloader::safe_send() - error: " << strerror(errno) << endl; - return SS_error; - } - } - return SS_success; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::safe_receive -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -int Downloader:: -safe_receive(int socket, DownloadStatus &status, int length, - long timeout, int &bytes) { - bytes = 0; - if (length == 0) { - downloader_cat.error() - << "Downloader::safe_receive() - requested 0 length receive!" << endl; - return RS_error; - } - struct timeval tv; - tv.tv_sec = timeout; - tv.tv_usec = 0; - fd_set rset; - FD_ZERO(&rset); - while (bytes < length) { - FD_SET(socket, &rset); - int sret = select(socket + 1, &rset, NULL, NULL, &tv); - if (sret == 0) { - downloader_cat.warning() - << "Downloader::safe_receive() - select timed out after: " - << timeout << " seconds" << endl; - return RS_timeout; - } else if (sret == -1) { - downloader_cat.error() - << "Downloader::safe_receive() - error: " << strerror(errno) << endl; - return RS_error; - } - int ret = recv(socket, status._next_in, length - bytes, 0); - if (ret > 0) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::safe_receive() - recv() got: " << ret << " bytes" - << endl; - bytes += ret; - status._next_in += ret; - status._bytes_in_buffer += ret; - if (bytes < length) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::safe_receive() - Download stalled" << endl; - _current_attempt_stalled = true; - } - } else if (ret == 0) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::safe_receive() - End of file" << endl; - return RS_eof; - } else { - downloader_cat.error() - << "Downloader::safe_receive() - error: " << strerror(errno) << endl; - return RS_error; - } - } - nassertr(bytes == length, RS_error); - return RS_success; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::attempt_read -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -int Downloader:: -attempt_read(int length, DownloadStatus &status, int &bytes_read) { - - bytes_read = 0; - for (int i = 0; i < downloader_timeout_retries; i++) { - - // Ensure we have enough room in the buffer to download length bytes - // If we don't have enough room, write the buffer to disk - if (status._bytes_in_buffer + length > _disk_buffer_size) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::attempt_read() - Flushing buffer" << endl; - - if (write_to_disk(status) == false) - return RS_error; - } - - // Make the request for length bytes - int bytes; - int ans = safe_receive(_socket, status, length, - (long)downloader_timeout, bytes); - bytes_read += bytes; - - switch (ans) { - case RS_error: - case RS_eof: - return ans; - case RS_timeout: - // Try again - break; - case RS_success: - nassertr(bytes == length, RS_error); - return RS_success; - default: - downloader_cat.error() - << "Downloader::attempt_read() - unknown return condition " - << "from safe_receive() : " << ans << endl; - return RS_error; - } - } - - // We timed out on retries consecutive attempts - this is considered - // a true timeout - return RS_timeout; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::download -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -int Downloader:: -download(const string &file_name, Filename file_dest, - const string &event_name, int first_byte, int last_byte, - int total_bytes, bool partial_content, bool sync, uint id) { - - if (_download_enabled == false) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - downloading is disabled" << endl; - return DS_abort; - } - - // Make sure we are still connected to the server - if (connect_to_server() == false) - return DS_abort; - - // Attempt to open the destination file - file_dest.set_binary(); - bool result; - if (partial_content == true && first_byte > 0) - result = file_dest.open_append(_dest_stream); - else - result = file_dest.open_write(_dest_stream); - if (result == false) { - downloader_cat.error() - << "Downloader::download() - Error opening file: " << file_dest - << " for writing" << endl; - return DS_abort; - } - - // Send an HTTP request for the file to the server - string request = "GET "; - request += file_name; - request += " HTTP/1.1\012Host: "; - request += _server_name; - request += "\012Connection: close"; - if (partial_content == true) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - Requesting byte range: " << first_byte - << "-" << last_byte << endl; - request += "\012Range: bytes="; - stringstream start_stream; - start_stream << first_byte << "-" << last_byte; - request += start_stream.str(); - } - request += "\012\012"; - int outlen = request.size(); - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - Sending request:\n" << request << endl; - int send_ret = safe_send(_socket, request.c_str(), outlen, - (long)downloader_timeout); - - // Handle timeouts on the send - if (send_ret == SS_timeout) { - for (int sr = 0; sr < downloader_timeout_retries; sr++) { - send_ret = safe_send(_socket, request.c_str(), outlen, - (long)downloader_timeout); - if (send_ret != SS_timeout) - break; - } - if (send_ret == SS_timeout) { - // We've really timed out - throw an event - downloader_cat.error() - << "Downloader::download() - send timed out after: " - << downloader_timeout_retries << " retries" << endl; - return DS_timeout; - } - } - - if (send_ret == SS_error) - return DS_abort; - - // Create a download status to maintain download progress information - DownloadStatus status(_buffer->_buffer, event_name, first_byte, last_byte, - total_bytes, partial_content, id); - bool got_any_data = false; - - // Loop at the requested frequency until the download completes - for (;;) { - bool resize_buffer = false; - - // Ensure that these don't change while we're computing read_size -#ifdef HAVE_IPC - _buffer_lock.lock(); -#endif - - nassertr(_frequency > 0, DS_abort); - // If byte rate has changed, recompute read size and write buffer size - if (_new_byte_rate > 0) { - _read_size = (int)ceil(_new_byte_rate * _frequency); - _byte_rate = _new_byte_rate; - _new_byte_rate = 0; - resize_buffer = true; - } - - // If the disk write frequency has changed, compute a new buffer size - if (_new_disk_write_frequency > 0) { - _disk_write_frequency = _new_disk_write_frequency; - _new_disk_write_frequency = 0; - resize_buffer = true; - } - - if (resize_buffer == true) { - // Flush the write buffer before resizing it - if (status._bytes_in_buffer > 0) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - Flushing buffer" << endl; - - if (write_to_disk(status) == false) { - downloader_cat.error() - << "Downloader::download() - failed to flush buffer during " - << "resize" << endl; - return DS_abort; - } - } - - // Resize the buffer - _disk_buffer_size = (_disk_write_frequency * _read_size); - - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - resizing disk buffer to: " - << _disk_buffer_size << endl; - _buffer.clear(); - downloader_cat.debug() - << "Downloader::download() - buffer cleared" << endl; - _buffer = new Buffer(_disk_buffer_size); - // Update the status with the new buffer - status._buffer = _buffer->_buffer; - status.reset(); - downloader_cat.debug() - << "Downloader::download() - new buffer created" << endl; - } - -#ifdef HAVE_IPC - _buffer_lock.unlock(); -#endif - - // Attempt to read - int bytes_read; - - int ret = attempt_read(_read_size, status, bytes_read); - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - stalled status: " << _current_attempt_stalled - << endl; - - _last_attempt_stalled = _current_attempt_stalled; - _current_attempt_stalled = false; - - if (bytes_read > 0) - got_any_data = true; - - switch (ret) { - case RS_error: - - downloader_cat.error() - << "Downloader::download() - Error reading from socket: " - << strerror(errno) << endl; - return DS_abort; - - case RS_timeout: - - { - // We've really timed out - throw an event - downloader_cat.error() - << "Downloader::download() - receive timed out after: " - << downloader_timeout_retries << " retries" << endl; - if (bytes_read > 0) { - if (write_to_disk(status) == false) { - downloader_cat.error() - << "Downloader::download() - write to disk failed after " - << "timeout!" << endl; - return DS_abort; - } - } - return DS_timeout; - } - - case RS_success: - - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - Got: " << bytes_read << " bytes" - << endl; - break; - - case RS_eof: - - { - // We occasionally will get 0 bytes on the first attempt - we - // don't want to treat this as end of file in any case - if (got_any_data == true) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Download for: " << file_name << " completed" << endl; - bool ret = true; - if (bytes_read > 0) - ret = write_to_disk(status); - _dest_stream.close(); - - // The "Connection: close" line tells server to close connection - // when the download is complete - _connected = false; - if (ret == false) - return DS_abort; - return DS_success; - } else { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::download() - Received 0 bytes" << endl; - } - } - break; - - default: - - downloader_cat.error() - << "Downloader::download() - Unknown return value from " - << "attempt_read() : " << ret << endl; - return DS_abort; - - } // switch(ret) - - // Sleep for the requested frequency - nap(); - - } // for (;;) - - downloader_cat.error() - << "Downloader::download() - Dropped out of for loop without returning!" - << endl; - return DS_abort; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::parse_http_response -// Access: Private -// Description: Check the HTTP response from the server -//////////////////////////////////////////////////////////////////// -bool Downloader:: -parse_http_response(const string &resp) { - size_t ws = resp.find(" ", 0); - string httpstr = resp.substr(0, ws); - if (!(httpstr == "HTTP/1.1")) { - downloader_cat.error() - << "Downloader::parse_http_response() - not HTTP/1.1 - got: " - << httpstr << endl; - return false; - } - size_t ws2 = resp.find(" ", ws); - string numstr = resp.substr(ws, ws2); - nassertr(numstr.length() > 0, false); - int num = atoi(numstr.c_str()); - switch (num) { - case 200: - case 206: - return true; - case 202: - // Accepted - server may not honor request, though - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::parse_http_response() - got a 202 Accepted - " - << "server does not guarantee to honor this request" << endl; - return true; - case 201: - case 203: - case 204: - case 205: - default: - break; - } - - downloader_cat.error() - << "Downloader::parse_http_response() - Invalid response: " - << resp << endl; - return false; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::parse_header -// Access: Private -// Description: Looks for a valid header. If it finds one, it -// calculates the header length and strips it from -// the download status structure. Function returns false -// on an error condition, otherwise true. -//////////////////////////////////////////////////////////////////// -bool Downloader:: -parse_header(DownloadStatus &status) { - - if (status._header_is_complete == true) - return true; - - if (status._bytes_in_buffer == 0) { - downloader_cat.error() - << "Downloader::parse_header() - Empty buffer!" << endl; - return false; - } - - string bufstr((char *)status._start, status._bytes_in_buffer); - size_t p = 0; - while (p < bufstr.length()) { - // Server sends out CR LF (\r\n) as newline delimiter - size_t nl = bufstr.find("\015\012", p); - if (nl == string::npos) { - downloader_cat.error() - << "Downloader::parse_header() - No newlines in buffer of " - << "length: " << status._bytes_in_buffer << endl; - return false; - } else if (p == 0 && nl == p) { - downloader_cat.error() - << "Downloader::parse_header() - Buffer begins with newline!" - << endl; - return false; - } - - string component = bufstr.substr(p, nl - p); - - // The first line of the response should say whether - // got an error or not - if (status._first_line_complete == false) { - status._first_line_complete = true; - if (parse_http_response(component) == true) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::parse_header() - Header is valid: " - << component << endl; - status._header_is_valid = true; - } else { - return false; - } - } - - // Look for content length - size_t cpos = component.find(":"); - string tline = component.substr(0, cpos); - if (status._partial_content == true && tline == "Content-Length") { - tline = component.substr(cpos + 2, string::npos); - int server_download_bytes = atoi(tline.c_str()); - int client_download_bytes = status._last_byte - status._first_byte; - if (status._first_byte == 0) - client_download_bytes += 1; - if (client_download_bytes != server_download_bytes) { - downloader_cat.error() - << "Downloader::parse_header() - server size = " - << server_download_bytes << ", client size = " - << client_download_bytes << " (" - << status._last_byte << "-" << status._first_byte << ")" << endl; - return false; - } - } - - // Two consecutive (CR LF)s indicates end of HTTP header - if (nl == p) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::parse_header() - Header is complete" << endl; - status._header_is_complete = true; - - // Strip the header out of the status buffer - int header_length = nl + 2; - status._start += header_length; - status._bytes_in_buffer -= header_length; - - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::parse_header() - Stripping out header of size: " - << header_length << endl; - - return true; - } - - p = nl + 2; - } - - if (status._header_is_complete == false) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::parse_header() - Reached end of buffer without " - << "successfully parsing the header - buffer size: " - << status._bytes_in_buffer << endl; - } - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::write_to_disk -// Access: Private -// Description: Writes a download to disk. If there is a header, -// the pointer and size are adjusted so the header -// is excluded. Function returns false on error -// condition. -//////////////////////////////////////////////////////////////////// -bool Downloader:: -write_to_disk(DownloadStatus &status) { - - // Ensure the header has been parsed successfully first - if (parse_header(status) == false) - return false; - - if (status._header_is_complete == false) { - downloader_cat.error() - << "Downloader::write_to_disk() - Incomplete HTTP header - " - << "(or header was larger than download buffer) - " - << "try increasing download-buffer-size" << endl; - return false; - } - - // Write what we have so far to disk - if (status._bytes_in_buffer > 0) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Downloader::write_to_disk() - Writing " - << status._bytes_in_buffer << " to disk" << endl; - - _dest_stream.write(status._start, status._bytes_in_buffer); - status._total_bytes_written += status._bytes_in_buffer; - - // Throw an event to indicate how many bytes have been written so far - if (!status._event_name.empty()) { - PT_Event write_event = new Event(status._event_name); - write_event->add_parameter(EventParameter((int)status._id)); - write_event->add_parameter(EventParameter(DS_write)); - write_event->add_parameter(EventParameter(status._total_bytes_written)); - throw_event(write_event); - } - } - - status.reset(); - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::DownloadStatus::constructor -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -Downloader::DownloadStatus:: -DownloadStatus(char *buffer, const string &event_name, int first_byte, - int last_byte, int total_bytes, bool partial_content, uint id) { - _first_line_complete = false; - _header_is_complete = false; - _header_is_valid = false; - _buffer = buffer; - _event_name = event_name; - _first_byte = first_byte; - _last_byte = last_byte; - _total_bytes = total_bytes; - _partial_content = partial_content; - _id = id; - reset(); -} - -//////////////////////////////////////////////////////////////////// -// Function: Downloader::DownloadStatus::reset -// Access: Public -// Description: Resets the status buffer for more downloading after -// a write. -//////////////////////////////////////////////////////////////////// -void Downloader::DownloadStatus:: -reset(void) { - _start = _buffer; - _next_in = _start; - _bytes_in_buffer = 0; - _total_bytes_written = 0; -} diff --git a/panda/src/downloader/asyncDownloader.h b/panda/src/downloader/asyncDownloader.h deleted file mode 100644 index 749012bab9..0000000000 --- a/panda/src/downloader/asyncDownloader.h +++ /dev/null @@ -1,150 +0,0 @@ -// Filename: asyncDownloader.h -// Created by: mike (09Jan97) -// -//////////////////////////////////////////////////////////////////// -// -// PANDA 3D SOFTWARE -// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved -// -// All use of this software is subject to the terms of the Panda 3d -// Software license. You should have received a copy of this license -// along with this source code; you will also find a current copy of -// the license at http://www.panda3d.org/license.txt . -// -// To contact the maintainers of this program write to -// panda3d@yahoogroups.com . -// -//////////////////////////////////////////////////////////////////// -#ifndef ASYNCDOWNLOADER_H -#define ASYNCDOWNLOADER_H -// -//////////////////////////////////////////////////////////////////// -// Includes -//////////////////////////////////////////////////////////////////// -#include -#include -#include -#include -#include -#include "asyncUtility.h" - -#if defined(WIN32_VC) - #include -#else - #include // Irix seems to require this one for resolv.h. - #include - #include - #include -#endif - -class DownloaderToken; - -//////////////////////////////////////////////////////////////////// -// Class : Downloader -// Description : -//////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS Downloader : public AsyncUtility { -PUBLISHED: - enum DownloadCode { - DS_write = 2, - DS_success = 1, - DS_abort = -1, - DS_timeout = -2 - }; - Downloader(void); - //Downloader(PT(Buffer) buffer); - virtual ~Downloader(void); - - bool connect_to_server(const string &name, uint port=80); - void disconnect_from_server(void); - - int request_sync_download(const string &file_name, const Filename &file_dest, - const string &event_name); - int request_sync_download(const string &file_name, const Filename &file_dest, - const string &event_name, int first_byte, - int last_byte, int total_bytes, - bool partial_content = true); - int request_download(const string &file_name, const Filename &file_dest, - const string &event_name, bool sync = false); - int request_download(const string &file_name, const Filename &file_dest, - const string &event_name, int first_byte, - int last_byte, int total_bytes, - bool partial_content = true, bool sync = false); - - INLINE void set_byte_rate(float bytes); - INLINE float get_byte_rate(void) const; - INLINE void set_disk_write_frequency(int frequency); - INLINE int get_disk_write_frequency(void) const; - INLINE void enable_download(bool val); - INLINE bool is_download_enabled(void) const; - INLINE bool get_last_attempt_stalled(void) const; - -private: - class DownloadStatus { - public: - DownloadStatus(char *buffer, const string &event_name, int first_byte, - int last_byte, int total_bytes, bool partial_content, - uint id); - void reset(void); - - public: - bool _first_line_complete; - bool _header_is_complete; - bool _header_is_valid; - char *_start; - char *_next_in; - int _bytes_in_buffer; - string _event_name; - int _total_bytes_written; - int _first_byte; - int _last_byte; - int _total_bytes; - bool _partial_content; - uint _id; - char *_buffer; - }; - - void init(); - int download(const string &file_name, Filename file_dest, - const string &event_name, int first_byte, - int last_byte, int total_bytes, bool partial_content, - bool sync, uint id); - virtual bool process_request(void); - bool parse_header(DownloadStatus &status); - bool write_to_disk(DownloadStatus &status); - bool connect_to_server(void); - int safe_send(int socket, const char *data, int length, long timeout); - int safe_receive(int socket, DownloadStatus &status, int length, - long timeout, int &bytes); - bool parse_http_response(const string &resp); - int attempt_read(int length, DownloadStatus &status, int &bytes_read); - - typedef TokenBoard DownloaderTokenBoard; - DownloaderTokenBoard *_token_board; - - bool _connected; - -#ifdef HAVE_IPC - mutex _buffer_lock; -#endif - - int _socket; - PT(Buffer) _buffer; - int _disk_write_frequency; - int _new_disk_write_frequency; - float _byte_rate; - float _new_byte_rate; - int _read_size; - bool _download_enabled; - ofstream _dest_stream; - int _disk_buffer_size; - bool _last_attempt_stalled; - bool _current_attempt_stalled; - - string _server_name; - struct sockaddr_in _sin; -}; - -#include "asyncDownloader.I" - -#endif diff --git a/panda/src/downloader/asyncExtractor.cxx b/panda/src/downloader/asyncExtractor.cxx deleted file mode 100644 index fc478fe60f..0000000000 --- a/panda/src/downloader/asyncExtractor.cxx +++ /dev/null @@ -1,262 +0,0 @@ -// Filename: asyncExtractor.cxx -// Created by: mike (09Jan97) -// -//////////////////////////////////////////////////////////////////// -// -// PANDA 3D SOFTWARE -// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved -// -// All use of this software is subject to the terms of the Panda 3d -// Software license. You should have received a copy of this license -// along with this source code; you will also find a current copy of -// the license at http://www.panda3d.org/license.txt . -// -// To contact the maintainers of this program write to -// panda3d@yahoogroups.com . -// -//////////////////////////////////////////////////////////////////// - -#include "config_downloader.h" - -#include -#include -#include -#include -#include - -#include "asyncExtractor.h" - -//////////////////////////////////////////////////////////////////// -// Defines -//////////////////////////////////////////////////////////////////// - -//////////////////////////////////////////////////////////////////// -// Class : ExtractorToken -// Description : Holds a request for the extractor. -//////////////////////////////////////////////////////////////////// -class ExtractorToken : public ReferenceCount { -public: - INLINE ExtractorToken(uint id, const Filename &source_file, - const string &event_name, - const Filename &rel_path) { - _id = id; - _source_file = source_file; - _event_name = event_name; - _rel_path = rel_path; - } - int _id; - Filename _source_file; - string _event_name; - Filename _rel_path; -}; - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Extractor:: -Extractor(void) : AsyncUtility() { - PT(Buffer) buffer = new Buffer(extractor_buffer_size); - init(buffer); -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Extractor:: -Extractor(PT(Buffer) buffer) : AsyncUtility() { - init(buffer); -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::Constructor -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -void Extractor:: -init(PT(Buffer) buffer) { - nassertv(!buffer.is_null()); - _frequency = extractor_frequency; - _token_board = new ExtractorTokenBoard; - _buffer = buffer; -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::Destructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Extractor:: -~Extractor(void) { - destroy_thread(); - - delete _token_board; -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::request_extract -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -int Extractor:: -request_extract(const Filename &source_file, const string &event_name, - const Filename &rel_path) { - - PT(ExtractorToken) tok; - if (_threads_enabled) { - - // Make sure we actually are threaded - if (!_threaded) { - downloader_cat.info() - << "Extractor::request_extract() - create_thread() was " - << "never called! Calling it now..." << endl; - create_thread(); - } - - // We need to grab the lock in order to signal the condition variable -#ifdef HAVE_IPC - _lock.lock(); -#endif - - if (_token_board->_waiting.is_full()) { - downloader_cat.error() - << "Extractor::request_extract() - Too many pending requests\n"; - return 0; - } - - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Extract requested for file: " << source_file << endl; - } - - tok = new ExtractorToken(_next_token++, source_file, event_name, - rel_path); - _token_board->_waiting.insert(tok); - -#ifdef HAVE_IPC - _request_cond->signal(); - _lock.unlock(); -#endif - - } else { - // If we're not running asynchronously, process the load request - // directly now. - if (_token_board->_waiting.is_full()) { - downloader_cat.error() - << "Extractor::request_extract() - Too many pending requests\n"; - return 0; - } - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Extract requested for file: " << source_file << endl; - } - - tok = new ExtractorToken(_next_token++, source_file, event_name, - rel_path); - _token_board->_waiting.insert(tok); - process_request(); - } - - return tok->_id; -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::process_request -// Access: Private -// Description: Serves any requests on the token board, moving them -// to the done queue. -//////////////////////////////////////////////////////////////////// -bool Extractor:: -process_request() { - if (_shutdown) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Extractor shutting down...\n"; - return false; - } - - // If there is actually a request token - process it - while (!_token_board->_waiting.is_empty()) { - PT(ExtractorToken) tok = _token_board->_waiting.extract(); - if (extract(tok->_source_file, tok->_rel_path)) { - _token_board->_done.insert(tok); - - // Throw a "done" event now. - if (!tok->_event_name.empty()) { - PT_Event done = new Event(tok->_event_name); - done->add_parameter(EventParameter((int)tok->_id)); - throw_event(done); - } - - if (downloader_cat.is_debug()) { - downloader_cat.debug() - << "Extractor::process_request() - extract complete for " - << tok->_source_file << "\n"; - } - } - } - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::extract -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -bool Extractor:: -extract(Filename &source_file, const Filename &rel_path) { - - // Open source file - ifstream read_stream; - source_file.set_binary(); - if (!source_file.open_read(read_stream)) { - downloader_cat.error() - << "Extractor::extract() - Error opening source file: " - << source_file << endl; - return false; - } - - // Determine source file length - read_stream.seekg(0, ios::end); - int source_file_length = read_stream.tellg(); - read_stream.seekg(0, ios::beg); - - // Read the multifile header - Multifile mfile; - - // Read from the source file and write to the appropriate extracted file - int total_bytes_read = 0; - bool read_all_input = false; - bool handled_all_input = false; - int source_buffer_length; - while (handled_all_input == false) { - - // See if there is anything left in the source file - if (read_all_input == false) { - read_stream.read(_buffer->_buffer, _buffer->get_length()); - source_buffer_length = read_stream.gcount(); - total_bytes_read += source_buffer_length; - if (read_stream.eof()) { - nassertr(total_bytes_read == source_file_length, false); - read_all_input = true; - } - } - - // Write to the out file - char *start = _buffer->_buffer; - int size = source_buffer_length; - if (mfile.write_extract(start, size, rel_path) == true) - handled_all_input = true; - - nap(); - } - - read_stream.close(); - source_file.unlink(); - - return true; -} diff --git a/panda/src/downloader/asyncExtractor.h b/panda/src/downloader/asyncExtractor.h deleted file mode 100644 index 42a7b69dda..0000000000 --- a/panda/src/downloader/asyncExtractor.h +++ /dev/null @@ -1,58 +0,0 @@ -// Filename: asyncExtractor.h -// Created by: mike (09Jan97) -// -//////////////////////////////////////////////////////////////////// -// -// PANDA 3D SOFTWARE -// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved -// -// All use of this software is subject to the terms of the Panda 3d -// Software license. You should have received a copy of this license -// along with this source code; you will also find a current copy of -// the license at http://www.panda3d.org/license.txt . -// -// To contact the maintainers of this program write to -// panda3d@yahoogroups.com . -// -//////////////////////////////////////////////////////////////////// -#ifndef ASYNCEXTRACTOR_H -#define ASYNCEXTRACTOR_H -// -//////////////////////////////////////////////////////////////////// -// Includes -//////////////////////////////////////////////////////////////////// -#include -#include -#include -#include -#include -#include "asyncUtility.h" - -class ExtractorToken; - -//////////////////////////////////////////////////////////////////// -// Class : Extractor -// Description : -//////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS Extractor : public AsyncUtility { -PUBLISHED: - Extractor(void); - Extractor(PT(Buffer) buffer); - virtual ~Extractor(void); - - int request_extract(const Filename &source_file, - const string &event_name, const Filename &rel_path = ""); - - bool extract(Filename &source_file, const Filename &rel_path); - -private: - void init(PT(Buffer) buffer); - virtual bool process_request(void); - - typedef TokenBoard ExtractorTokenBoard; - ExtractorTokenBoard *_token_board; - - PT(Buffer) _buffer; -}; - -#endif diff --git a/panda/src/downloader/extractor.I b/panda/src/downloader/extractor.I index 36a39e9a57..4b9e94ab28 100644 --- a/panda/src/downloader/extractor.I +++ b/panda/src/downloader/extractor.I @@ -21,16 +21,11 @@ //////////////////////////////////////////////////////////////////// // Function: Extractor::get_progress // Access: Public -// Description: +// Description: Returns the fraction of the Multifile extracted so +// far. //////////////////////////////////////////////////////////////////// INLINE float Extractor:: -get_progress(void) const { - if (_initiated == false) { - downloader_cat.warning() - << "Extractor::get_progress() - Extraction has not been initiated" - << endl; - return 0.0; - } - nassertr(_source_file_length > 0, 0.0); - return ((float)_total_bytes_read / (float)_source_file_length); +get_progress() const { + nassertr(_initiated, 0.0f); + return ((float)_subfile_index / (float)(_multifile.get_num_subfiles() + 1)); } diff --git a/panda/src/downloader/extractor.cxx b/panda/src/downloader/extractor.cxx index 56dbf974c6..8de908c49a 100644 --- a/panda/src/downloader/extractor.cxx +++ b/panda/src/downloader/extractor.cxx @@ -16,17 +16,12 @@ // //////////////////////////////////////////////////////////////////// +#include "extractor.h" #include "config_downloader.h" -#include -#include +#include "filename.h" +#include "error_utils.h" -#include -#include "extractor.h" - -//////////////////////////////////////////////////////////////////// -// Defines -//////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////// // Function: Extractor::Constructor @@ -34,35 +29,8 @@ // Description: //////////////////////////////////////////////////////////////////// Extractor:: -Extractor(void) { - PT(Buffer) buffer = new Buffer(extractor_buffer_size); - init(buffer); -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Extractor:: -Extractor(PT(Buffer) buffer) { - init(buffer); -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::Constructor -// Access: Private -// Description: -//////////////////////////////////////////////////////////////////// -void Extractor:: -init(PT(Buffer) buffer) { - if (downloader_cat.is_debug()) - downloader_cat.debug() - << "Extractor constructor called" << endl; +Extractor() { _initiated = false; - nassertv(!buffer.is_null()); - _buffer = buffer; - _mfile = NULL; } //////////////////////////////////////////////////////////////////// @@ -71,136 +39,163 @@ init(PT(Buffer) buffer) { // Description: //////////////////////////////////////////////////////////////////// Extractor:: -~Extractor(void) { +~Extractor() { if (downloader_cat.is_debug()) downloader_cat.debug() << "Extractor destructor called" << endl; - if (_initiated == true) + if (_initiated) { cleanup(); + } } //////////////////////////////////////////////////////////////////// // Function: Extractor::initiate // Access: Public -// Description: +// Description: Begins the extraction process. Returns EU_success if +// successful, EU_error_abort otherwise. +// +// After calling initiate(), you should repeatedly call +// run() as a background task until the file is +// completely extracted. //////////////////////////////////////////////////////////////////// int Extractor:: -initiate(Filename &source_file, const Filename &rel_path) { - - if (_initiated == true) { +initiate(const Filename &multifile_name, const Filename &extract_to) { + if (_initiated) { downloader_cat.error() << "Extractor::initiate() - Extraction has already been initiated" << endl; return EU_error_abort; } - // Open source file - _source_file = source_file; - _source_file.set_binary(); - if (!_source_file.open_read(_read_stream)) { + _multifile_name = multifile_name; + _extract_to = extract_to; + + if (!_multifile.open_read(_multifile_name)) { downloader_cat.error() - << "Extractor::extract() - Error opening source file: " - << _source_file << " : " << strerror(errno) << endl; - return get_write_error(); + << "Error opening Multifile " << _multifile_name << ".\n"; + return EU_error_abort; } - _rel_path = rel_path; - - // Determine source file length - _read_stream.seekg(0, ios::end); - _source_file_length = _read_stream.tellg(); - _read_stream.seekg(0, ios::beg); - - _total_bytes_read = 0; - _read_all_input = false; - _handled_all_input = false; - _mfile = new Multifile(); + _subfile_index = 0; + _subfile_pos = 0; + _subfile_length = 0; + _read = (istream *)NULL; _initiated = true; + return EU_success; } +//////////////////////////////////////////////////////////////////// +// Function: Extractor::run +// Access: Public +// Description: Extracts the next small unit of data from the +// Multifile. Returns EU_ok if progress is continuing, +// EU_error_abort if there is a problem, or EU_success +// when the last piece has been extracted. +//////////////////////////////////////////////////////////////////// +int Extractor:: +run() { + if (!_initiated) { + downloader_cat.error() + << "Extractor::run() - Extraction has not been initiated" + << endl; + return EU_error_abort; + } + + if (_read == (istream *)NULL) { + // Time to open the next subfile. + if (_subfile_index >= _multifile.get_num_subfiles()) { + // All done! + cleanup(); + return EU_success; + } + + Filename subfile_filename(_extract_to, + _multifile.get_subfile_name(_subfile_index)); + subfile_filename.set_binary(); + subfile_filename.make_dir(); + if (!subfile_filename.open_write(_write)) { + downloader_cat.error() + << "Unable to write to " << subfile_filename << ".\n"; + cleanup(); + return EU_error_abort; + } + + _subfile_length = _multifile.get_subfile_length(_subfile_index); + _subfile_pos = 0; + _read = &_multifile.open_read_subfile(_subfile_index); + + } else if (_subfile_pos >= _subfile_length) { + // Time to close this subfile. + _multifile.close_subfile(); + _write.close(); + _read = (istream *)NULL; + _subfile_index++; + + } else { + // Read a number of bytes from the subfile and write them to the + // output. + size_t max_bytes = min((size_t)extractor_buffer_size, + _subfile_length - _subfile_pos); + for (size_t p = 0; p < max_bytes; p++) { + int byte = _read->get(); + if (_read->eof() || _read->fail()) { + downloader_cat.error() + << "Unexpected EOF on multifile " << _multifile_name << ".\n"; + cleanup(); + return EU_error_abort; + } + _write.put(byte); + } + _subfile_pos += max_bytes; + } + + return EU_ok; +} + +//////////////////////////////////////////////////////////////////// +// Function: Extractor::extract +// Access: Public +// Description: A convenience function to extract the Multifile all +// at once, when you don't care about doing it in the +// background. +//////////////////////////////////////////////////////////////////// +bool Extractor:: +extract(const Filename &source_file, const Filename &rel_path) { + int ret = initiate(source_file, rel_path); + if (ret < 0) { + return false; + } + for (;;) { + ret = run(); + if (ret == EU_success) { + return true; + } + if (ret < 0) { + return false; + } + } + return false; +} + //////////////////////////////////////////////////////////////////// // Function: Extractor::cleanup // Access: Private // Description: //////////////////////////////////////////////////////////////////// void Extractor:: -cleanup(void) { +cleanup() { if (downloader_cat.is_debug()) downloader_cat.debug() << "Extractor cleanup called" << endl; - if (_initiated == false) { + if (!_initiated) { downloader_cat.error() << "Extractor::cleanup() - Extraction has not been initiated" << endl; return; } - delete _mfile; - _mfile = NULL; - _read_stream.close(); -// _source_file.unlink(); - _initiated = false; -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::run -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -int Extractor:: -run(void) { - if (_initiated == false) { - downloader_cat.error() - << "Extractor::run() - Extraction has not been initiated" - << endl; - return EU_error_abort; - } - - // See if there is anything left in the source file - if (_read_all_input == false) { - _read_stream.read(_buffer->_buffer, _buffer->get_length()); - _source_buffer_length = _read_stream.gcount(); - _total_bytes_read += _source_buffer_length; - if (_read_stream.eof()) { - nassertr(_total_bytes_read == _source_file_length, false); - _read_all_input = true; - } - } - - char *buffer_start = _buffer->_buffer; - int buffer_size = _source_buffer_length; - - // Write to the out file - int write_ret = _mfile->write(buffer_start, buffer_size, _rel_path); - if (write_ret == EU_success) { - cleanup(); - return EU_success; - } else if (write_ret < 0) { - downloader_cat.error() - << "Extractor::run() - got error from Multifile: " << write_ret - << endl; - return write_ret; - } - return EU_ok; -} - -//////////////////////////////////////////////////////////////////// -// Function: Extractor::extract -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -bool Extractor:: -extract(Filename &source_file, const Filename &rel_path) { - int ret = initiate(source_file, rel_path); - if (ret < 0) - return false; - for (;;) { - ret = run(); - if (ret == EU_success) - return true; - if (ret < 0) - return false; - } - return false; + _multifile.close(); + _write.close(); + _read = (istream *)NULL; } diff --git a/panda/src/downloader/extractor.h b/panda/src/downloader/extractor.h index 8079991aa6..757a620c24 100644 --- a/panda/src/downloader/extractor.h +++ b/panda/src/downloader/extractor.h @@ -17,49 +17,51 @@ //////////////////////////////////////////////////////////////////// #ifndef EXTRACTOR_H #define EXTRACTOR_H -// -//////////////////////////////////////////////////////////////////// -// Includes -//////////////////////////////////////////////////////////////////// -#include -#include -#include -#include -#include + +#include "pandabase.h" +#include "filename.h" +#include "buffer.h" +#include "multifile.h" +#include "pointerTo.h" //////////////////////////////////////////////////////////////////// // Class : Extractor -// Description : +// Description : This class automatically extracts the contents of a +// Multifile to the current directory (or to a specified +// directory) in the background. +// +// It is designed to limit its use of system resources +// and run unobtrusively in the background. After +// initiate(), each call to run() extracts another small +// portion of the Multifile. Call run() repeatedly +// whenever you have spare cycles until run() returns +// EU_success. //////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS Extractor { PUBLISHED: - Extractor(void); - Extractor(PT(Buffer) buffer); - virtual ~Extractor(void); + Extractor(); + ~Extractor(); - int initiate(Filename &source_file, const Filename &rel_path = ""); - int run(void); + int initiate(const Filename &multifile_name, const Filename &extract_to = ""); + int run(); - bool extract(Filename &source_file, const Filename &rel_path = ""); + bool extract(const Filename &multifile_name, const Filename &extract_to = ""); INLINE float get_progress(void) const; private: - void init(PT(Buffer) buffer); - void cleanup(void); + void cleanup(); bool _initiated; - PT(Buffer) _buffer; + Filename _multifile_name; + Filename _extract_to; + Multifile _multifile; - ifstream _read_stream; - int _source_file_length; - Multifile *_mfile; - int _total_bytes_read; - bool _read_all_input; - bool _handled_all_input; - int _source_buffer_length; - Filename _source_file; - Filename _rel_path; + int _subfile_index; + size_t _subfile_pos; + size_t _subfile_length; + istream *_read; + ofstream _write; }; #include "extractor.I" diff --git a/panda/src/downloadertools/multify.cxx b/panda/src/downloadertools/multify.cxx index 9dc61c041e..e85b866c89 100644 --- a/panda/src/downloadertools/multify.cxx +++ b/panda/src/downloadertools/multify.cxx @@ -16,89 +16,354 @@ // //////////////////////////////////////////////////////////////////// -#include +#include "pandabase.h" #ifndef HAVE_GETOPT - #include + #include "gnu_getopt.h" #else #include #endif -#include -#include -#ifndef OLD_WAY - #include -#endif +#include "multifile.h" +#include "filename.h" +#include + + +bool create = false; // -c +bool append = false; // -r +bool list = false; // -t +bool extract = false; // -x +bool verbose = false; // -v +Filename multifile_name; // -f +bool got_multifile_name = false; +bool to_stdout = false; // -O +Filename chdir_to; // -C +bool got_chdir_to = false; +size_t scale_factor = 0; // -F + +void +usage() { + cerr << "Usage: multify -[c|r|t|x] -f [options] ...\n"; +} + +void +help() { + cerr << "multify is used to store and extract files from a Panda Multifile.\n" + << "This is similar to a tar or zip file in that it is an archive file that\n" + << "contains a number of subfiles that may later be extracted.\n\n" + + << "The command-line options for multify are designed to be similar to those\n" + << "for tar, the traditional Unix archiver utility.\n\n"; + usage(); +} + +bool +is_named(const string &subfile_name, int argc, char *argv[]) { + // Returns true if the indicated subfile appears on the list of + // files named on the command line. + if (argc < 2) { + // No named files; everything is listed. + return true; + } + + for (int i = 1; i < argc; i++) { + if (subfile_name == argv[i]) { + return true; + } + } + + return false; +} + +bool +add_directory(Multifile &multifile, const Filename &directory_name) { + vector_string files; + if (!directory_name.scan_directory(files)) { + cerr << "Unable to scan directory " << directory_name << "\n"; + return false; + } + + bool okflag = true; + + vector_string::const_iterator fi; + for (fi = files.begin(); fi != files.end(); ++fi) { + Filename subfile_name(directory_name, (*fi)); + if (subfile_name.is_directory()) { + okflag = add_directory(multifile, subfile_name); + + } else if (!subfile_name.exists()) { + cerr << "Not found: " << subfile_name << "\n"; + okflag = false; + + } else { + if (verbose) { + cout << subfile_name << "\n"; + } + if (!multifile.add_subfile(subfile_name, subfile_name)) { + cerr << "Unable to add " << subfile_name << ".\n"; + okflag = false; + } + } + } + + return okflag; +} + +bool +add_files(int argc, char *argv[]) { + Multifile multifile; + if (append) { + if (!multifile.open_read_write(multifile_name)) { + cerr << "Unable to open " << multifile_name << " for updating.\n"; + return false; + } + } else { + if (!multifile.open_write(multifile_name)) { + cerr << "Unable to open " << multifile_name << " for writing.\n"; + return false; + } + } + + if (scale_factor != 0 && scale_factor != multifile.get_scale_factor()) { + cerr << "Setting scale factor to " << scale_factor << "\n"; + multifile.set_scale_factor(scale_factor); + } + + bool okflag = true; + for (int i = 1; i < argc; i++) { + Filename subfile_name = Filename::from_os_specific(argv[i]); + if (subfile_name.is_directory()) { + if (!add_directory(multifile, subfile_name)) { + okflag = false; + } + + } else if (!subfile_name.exists()) { + cerr << "Not found: " << subfile_name << "\n"; + okflag = false; + + } else { + if (verbose) { + cout << subfile_name << "\n"; + } + if (!multifile.add_subfile(subfile_name, subfile_name)) { + cerr << "Unable to add " << subfile_name << ".\n"; + okflag = false; + } + } + } + + if (multifile.needs_repack()) { + if (!multifile.repack()) { + cerr << "Failed to write " << multifile_name << ".\n"; + okflag = false; + } + } else { + if (!multifile.flush()) { + cerr << "Failed to write " << multifile_name << ".\n"; + okflag = false; + } + } + + return okflag; +} + +bool +extract_files(int argc, char *argv[]) { + if (!multifile_name.exists()) { + cerr << multifile_name << " not found.\n"; + return false; + } + Multifile multifile; + if (!multifile.open_read(multifile_name)) { + cerr << "Unable to open " << multifile_name << " for reading.\n"; + return false; + } + + int num_subfiles = multifile.get_num_subfiles(); + + for (int i = 0; i < num_subfiles; i++) { + string subfile_name = multifile.get_subfile_name(i); + if (is_named(subfile_name, argc, argv)) { + Filename filename = subfile_name; + if (got_chdir_to) { + filename = Filename(chdir_to, subfile_name); + } + if (to_stdout) { + if (verbose) { + cerr << filename << "\n"; + } + multifile.extract_subfile_to(i, cout); + } else { + if (verbose) { + cout << filename << "\n"; + } + multifile.extract_subfile(i, filename); + } + } + } + + return true; +} + +bool +list_files(int argc, char *argv[]) { + if (!multifile_name.exists()) { + cerr << multifile_name << " not found.\n"; + return false; + } + Multifile multifile; + if (!multifile.open_read(multifile_name)) { + cerr << "Unable to open " << multifile_name << " for reading.\n"; + return false; + } + + int num_subfiles = multifile.get_num_subfiles(); + + if (verbose) { + cout << num_subfiles << " subfiles:\n" << flush; + for (int i = 0; i < num_subfiles; i++) { + string subfile_name = multifile.get_subfile_name(i); + if (is_named(subfile_name, argc, argv)) { + printf("%12d %s\n", + multifile.get_subfile_length(i), + subfile_name.c_str()); + } + } + fflush(stdout); + if (multifile.get_scale_factor() != 1) { + cout << "Scale factor is " << multifile.get_scale_factor() << "\n"; + } + if (multifile.needs_repack()) { + cout << "Multifile needs to be repacked.\n"; + } + } else { + for (int i = 0; i < num_subfiles; i++) { + string subfile_name = multifile.get_subfile_name(i); + if (is_named(subfile_name, argc, argv)) { + cout << subfile_name << "\n"; + } + } + } + + return true; +} int main(int argc, char *argv[]) { - if (argc < 3) { - cerr << "Usage: multify -[x,c|vr] ..." << endl; - return 0; + if (argc < 2) { + usage(); + return 1; } - bool verbose = true; - bool extract = false; + // To emulate tar, we assume an implicit hyphen in front of the + // first argument if there is not one already. + if (argc >= 2) { + if (*argv[1] != '-' && *argv[1] != '\0') { + char *new_arg = new char[strlen(argv[1]) + 2]; + new_arg[0] = '-'; + strcpy(new_arg + 1, argv[1]); + argv[1] = new_arg; + } + } extern char *optarg; extern int optind; - int flag = getopt(argc, argv, "xcvr:"); + static const char *optflags = "crtxvf:OC:F:h"; + int flag = getopt(argc, argv, optflags); Filename rel_path; while (flag != EOF) { switch (flag) { - case 'x': - extract = true; - break; - case 'c': - break; - case 'v': - verbose = true; - break; - case 'r': - rel_path = optarg; - break; - default: - cerr << "Unhandled switch: " << flag << endl; - break; + case 'c': + create = true; + break; + case 'r': + append = true; + break; + case 't': + list = true; + break; + case 'x': + extract = true; + break; + case 'v': + verbose = true; + break; + case 'f': + multifile_name = Filename::from_os_specific(optarg); + got_multifile_name = true; + break; + case 'C': + chdir_to = Filename::from_os_specific(optarg); + got_chdir_to = true; + break; + case 'O': + to_stdout = true; + break; + case 'F': + { + char *endptr; + scale_factor = strtol(optarg, &endptr, 10); + if (*endptr != '\0') { + cerr << "Invalid integer: " << optarg << "\n"; + usage(); + return 1; + } + if (scale_factor == 0) { + cerr << "Scale factor must be nonzero.\n"; + usage(); + return 1; + } + } + break; + + case 'h': + help(); + return 1; + case '?': + usage(); + return 1; + default: + cerr << "Unhandled switch: " << flag << endl; + break; } - flag = getopt(argc, argv, "xcvr:"); + flag = getopt(argc, argv, optflags); } argc -= (optind - 1); argv += (optind - 1); - if (argc <= 1) { - cerr << "Usage: multify -[x,c|v] ..." << endl; - return 0; + // We should have exactly one of these options. + if ((create?1:0) + (append?1:0) + (list?1:0) + (extract?1:0) != 1) { + cerr << "Exactly one of -c, -r, -t, -x must be specified.\n"; + usage(); + return 1; } - Filename dest_file = argv[1]; - dest_file.set_binary(); - - if (verbose == true) { - if (extract == true) - cerr << "Extracting from: " << dest_file << endl; - else - cerr << "Appending to: " << dest_file << endl; - } - - Multifile mfile; - - if (extract == false) { - for (int i = 2; i < argc; i++) { - Filename in_file = argv[i]; - in_file.set_binary(); - mfile.add(in_file); + if (!got_multifile_name) { + if (argc <= 1) { + usage(); + return 1; } - if (mfile.write(dest_file) == false) - cerr << "Failed to write: " << dest_file << endl; - } else { -#ifdef OLD_WAY - mfile.read(dest_file); - mfile.extract_all(rel_path); -#else - Extractor extor; - extor.extract(dest_file, rel_path); -#endif + // For now, we allow -f to be omitted, and use the first argument + // as the archive name, for backward compatibility. Later we will + // remove this. + multifile_name = Filename::from_os_specific(argv[1]); + cerr << "Warning: using " << multifile_name + << " as archive name. Use -f in the future to specify this.\n"; + argc--; + argv++; } - return 1; + bool okflag = true; + if (create || append) { + okflag = add_files(argc, argv); + } else if (extract) { + okflag = extract_files(argc, argv); + } else { // list + okflag = list_files(argc, argv); + } + + if (okflag) { + return 0; + } else { + return 1; + } } diff --git a/panda/src/express/Sources.pp b/panda/src/express/Sources.pp index 8e5aa45dcc..eb9c0b9063 100644 --- a/panda/src/express/Sources.pp +++ b/panda/src/express/Sources.pp @@ -9,55 +9,61 @@ #define COMBINED_SOURCES $[TARGET]_composite1.cxx $[TARGET]_composite2.cxx #define SOURCES \ - bigEndian.h buffer.I buffer.h \ - checksumHashGenerator.I checksumHashGenerator.h circBuffer.I \ - circBuffer.h clockObject.I clockObject.h config_express.h \ - datagram.I datagram.h datagramGenerator.I \ - datagramGenerator.h datagramInputFile.I datagramInputFile.h \ - datagramIterator.I datagramIterator.h datagramOutputFile.I \ - datagramOutputFile.h datagramSink.I datagramSink.h \ - dcast.T dcast.h \ - error_utils.h \ - get_config_path.h hashGeneratorBase.I hashGeneratorBase.h \ - hashVal.I hashVal.h indent.I indent.h littleEndian.h \ - memoryInfo.I memoryInfo.h \ - memoryUsage.I memoryUsage.h \ - memoryUsagePointerCounts.I memoryUsagePointerCounts.h \ - memoryUsagePointers.I memoryUsagePointers.h \ - multifile.I multifile.h namable.I \ - namable.h nativeNumericData.I nativeNumericData.h \ - numeric_types.h pointerTo.I pointerTo.h \ - pointerToArray.I pointerToArray.h \ - profileTimer.I profileTimer.h \ - pta_uchar.h referenceCount.I referenceCount.h \ - register_type.I register_type.h \ - reversedNumericData.I reversedNumericData.h tokenBoard.I \ - tokenBoard.h trueClock.I trueClock.h typeHandle.I \ - typeHandle.h typedObject.I typedObject.h \ - typedReferenceCount.I typedReferenceCount.h typedef.h \ - typeRegistry.I typeRegistry.h \ - typeRegistryNode.I typeRegistryNode.h \ - vector_uchar.h \ - $[if $[HAVE_CRYPTO], \ - crypto_utils.cxx crypto_utils.h patchfile.I \ - patchfile.cxx patchfile.h ] + bigEndian.h buffer.I buffer.h \ + checksumHashGenerator.I checksumHashGenerator.h circBuffer.I \ + circBuffer.h clockObject.I clockObject.h config_express.h \ + datagram.I datagram.h datagramGenerator.I \ + datagramGenerator.h datagramInputFile.I datagramInputFile.h \ + datagramIterator.I datagramIterator.h datagramOutputFile.I \ + datagramOutputFile.h datagramSink.I datagramSink.h \ + dcast.T dcast.h \ + error_utils.h \ + get_config_path.h hashGeneratorBase.I hashGeneratorBase.h \ + hashVal.I hashVal.h indent.I indent.h \ + indirectLess.I indirectLess.h \ + littleEndian.h \ + memoryInfo.I memoryInfo.h \ + memoryUsage.I memoryUsage.h \ + memoryUsagePointerCounts.I memoryUsagePointerCounts.h \ + memoryUsagePointers.I memoryUsagePointers.h \ + multifile.I multifile.h namable.I \ + namable.h nativeNumericData.I nativeNumericData.h \ + numeric_types.h \ + ordered_vector.h ordered_vector.I ordered_vector.T \ + pointerTo.I pointerTo.h \ + pointerToArray.I pointerToArray.h \ + profileTimer.I profileTimer.h \ + pta_uchar.h referenceCount.I referenceCount.h \ + register_type.I register_type.h \ + reversedNumericData.I reversedNumericData.h tokenBoard.I \ + tokenBoard.h trueClock.I trueClock.h typeHandle.I \ + typeHandle.h typedObject.I typedObject.h \ + typedReferenceCount.I typedReferenceCount.h typedef.h \ + typeRegistry.I typeRegistry.h \ + typeRegistryNode.I typeRegistryNode.h \ + vector_uchar.h \ + $[if $[HAVE_CRYPTO], \ + crypto_utils.cxx crypto_utils.h patchfile.I \ + patchfile.cxx patchfile.h ] #define INCLUDED_SOURCES \ - buffer.cxx checksumHashGenerator.cxx clockObject.cxx \ - config_express.cxx datagram.cxx datagramGenerator.cxx \ - datagramInputFile.cxx datagramIterator.cxx \ - datagramOutputFile.cxx datagramSink.cxx dcast.cxx error_utils.cxx \ - get_config_path.cxx \ - hashGeneratorBase.cxx hashVal.cxx indent.cxx \ - memoryInfo.cxx memoryUsage.cxx memoryUsagePointerCounts.cxx \ - memoryUsagePointers.cxx multifile.cxx namable.cxx \ - nativeNumericData.cxx profileTimer.cxx \ - pta_uchar.cxx referenceCount.cxx register_type.cxx \ - reversedNumericData.cxx trueClock.cxx typeHandle.cxx \ - typedObject.cxx typedReferenceCount.cxx \ - typeRegistry.cxx typeRegistryNode.cxx vector_uchar.cxx + buffer.cxx checksumHashGenerator.cxx clockObject.cxx \ + config_express.cxx datagram.cxx datagramGenerator.cxx \ + datagramInputFile.cxx datagramIterator.cxx \ + datagramOutputFile.cxx datagramSink.cxx dcast.cxx error_utils.cxx \ + get_config_path.cxx \ + hashGeneratorBase.cxx hashVal.cxx indent.cxx \ + memoryInfo.cxx memoryUsage.cxx memoryUsagePointerCounts.cxx \ + memoryUsagePointers.cxx multifile.cxx namable.cxx \ + nativeNumericData.cxx \ + ordered_vector.cxx \ + profileTimer.cxx \ + pta_uchar.cxx referenceCount.cxx register_type.cxx \ + reversedNumericData.cxx trueClock.cxx typeHandle.cxx \ + typedObject.cxx typedReferenceCount.cxx \ + typeRegistry.cxx typeRegistryNode.cxx vector_uchar.cxx - #define INSTALL_HEADERS \ + #define INSTALL_HEADERS \ bigEndian.h buffer.I buffer.h checksumHashGenerator.I \ checksumHashGenerator.h circBuffer.I circBuffer.h clockObject.I \ clockObject.h config_express.h datagram.I datagram.h \ @@ -66,12 +72,15 @@ datagramOutputFile.I datagramOutputFile.h datagramSink.I \ datagramSink.h dcast.T dcast.h \ error_utils.h get_config_path.h hashGeneratorBase.I \ - hashGeneratorBase.h hashVal.I hashVal.h indent.I indent.h \ + hashGeneratorBase.h hashVal.I hashVal.h \ + indent.I indent.h \ + indirectLess.I indirectLess.h \ littleEndian.h memoryInfo.I memoryInfo.h memoryUsage.I \ memoryUsage.h memoryUsagePointerCounts.I \ memoryUsagePointerCounts.h memoryUsagePointers.I \ memoryUsagePointers.h multifile.I multifile.h namable.I namable.h \ nativeNumericData.I nativeNumericData.h numeric_types.h \ + ordered_vector.h ordered_vector.I ordered_vector.T \ patchfile.I patchfile.h pointerTo.I pointerTo.h \ pointerToArray.I pointerToArray.h profileTimer.I \ profileTimer.h pta_uchar.h referenceCount.I referenceCount.h \ @@ -98,3 +107,14 @@ #end test_bin_target + +#begin test_bin_target + #define TARGET test_ordered_vector + + #define SOURCES \ + test_ordered_vector.cxx + + #define LOCAL_LIBS $[LOCAL_LIBS] putil + #define OTHER_LIBS $[OTHER_LIBS] pystub + +#end test_bin_target diff --git a/panda/src/express/express_composite2.cxx b/panda/src/express/express_composite2.cxx index 01046a91eb..6b70e2d198 100644 --- a/panda/src/express/express_composite2.cxx +++ b/panda/src/express/express_composite2.cxx @@ -2,6 +2,7 @@ #include "multifile.cxx" #include "namable.cxx" #include "nativeNumericData.cxx" +#include "ordered_vector.cxx" #include "profileTimer.cxx" #include "pta_uchar.cxx" #include "referenceCount.cxx" diff --git a/panda/src/putil/indirectLess.I b/panda/src/express/indirectLess.I similarity index 100% rename from panda/src/putil/indirectLess.I rename to panda/src/express/indirectLess.I diff --git a/panda/src/putil/indirectLess.h b/panda/src/express/indirectLess.h similarity index 98% rename from panda/src/putil/indirectLess.h rename to panda/src/express/indirectLess.h index 0a1fbe1775..9d37495cf1 100644 --- a/panda/src/putil/indirectLess.h +++ b/panda/src/express/indirectLess.h @@ -19,7 +19,7 @@ #ifndef INDIRECTLESS_H #define INDIRECTLESS_H -#include +#include "pandabase.h" //////////////////////////////////////////////////////////////////// // Class : IndirectLess diff --git a/panda/src/express/multifile.I b/panda/src/express/multifile.I index b0986d272b..99e8c572da 100644 --- a/panda/src/express/multifile.I +++ b/panda/src/express/multifile.I @@ -16,27 +16,180 @@ // //////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////// -// Function: Multifile::write_header -// Access: Private -// Description: Writes the header uncompressed with platform- -// independent byte ordering +// Function: Multifile::is_read_valid +// Access: Published +// Description: Returns true if the Multifile has been opened for +// read mode and there have been no errors, and +// individual Subfile contents may be extracted. //////////////////////////////////////////////////////////////////// -INLINE void Multifile:: -write_header(ofstream &write_stream) { - _datagram.clear(); - _datagram.add_uint32(_magic_number); - _datagram.add_int32(_num_mfiles); - string msg = _datagram.get_message(); - write_stream.write((char *)msg.data(), msg.length()); +INLINE bool Multifile:: +is_read_valid() const { + return (_read != (istream *)NULL && !_read->fail() && + _open_subfile == (Subfile *)NULL); } //////////////////////////////////////////////////////////////////// -// Function: Multifile::get_header_length -// Access: Public -// Description: +// Function: Multifile::is_write_valid +// Access: Published +// Description: Returns true if the Multifile has been opened for +// write mode and there have been no errors, and +// Subfiles may be added or removed from the Multifile. //////////////////////////////////////////////////////////////////// -INLINE int Multifile:: -get_header_length(void) const { - return _header_length; +INLINE bool Multifile:: +is_write_valid() const { + return (_write != (ostream *)NULL && !_write->fail() && + _open_subfile == (Subfile *)NULL); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::needs_repack +// Access: Published +// Description: Returns true if the Multifile index is suboptimal and +// should be repacked. Call repack() to achieve this. +//////////////////////////////////////////////////////////////////// +INLINE bool Multifile:: +needs_repack() const { + return _needs_repack; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::set_scale_factor +// Access: Published +// Description: Changes the internal scale factor for this Multifile. +// +// This is normally 1, but it may be set to any +// arbitrary value (greater than zero) to support +// Multifile archives that exceed 4GB, if necessary. +// (Individual subfiles may still not exceed 4GB.) +// +// All addresses within the file are rounded up to the +// next multiple of _scale_factor, and zeros are written +// to the file to fill the resulting gaps. Then the +// address is divided by _scale_factor and written out +// as a 32-bit integer. Thus, setting a scale factor of +// 2 supports up to 8GB files, 3 supports 12GB files, +// etc. +// +// Calling this function on an already-existing +// Multifile forces an immediate repack() operation. +//////////////////////////////////////////////////////////////////// +INLINE void Multifile:: +set_scale_factor(size_t scale_factor) { + nassertv(scale_factor != 0); + if (_scale_factor != scale_factor) { + _scale_factor = scale_factor; + if (_next_index != 0) { + repack(); + } + } +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::get_scale_factor +// Access: Published +// Description: Returns the internal scale factor for this Multifile. +// See set_scale_factor(). +//////////////////////////////////////////////////////////////////// +INLINE size_t Multifile:: +get_scale_factor() const { + return _scale_factor; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::word_to_streampos +// Access: Private +// Description: Converts a size_t address read from the file to +// a streampos byte address within the file. +//////////////////////////////////////////////////////////////////// +INLINE streampos Multifile:: +word_to_streampos(size_t word) const { + return (streampos)word * (streampos)_scale_factor; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::streampos_to_word +// Access: Private +// Description: Converts a streampos byte address within the file to +// a size_t value suitable for writing to the file. +//////////////////////////////////////////////////////////////////// +INLINE size_t Multifile:: +streampos_to_word(streampos fpos) const { + return (size_t)((fpos + _scale_factor - 1) / _scale_factor); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::normalize_streampos +// Access: Private +// Description: Rounds the streampos byte address up to the next +// multiple of _scale_factor. Only multiples of +// _scale_factor may be written to the file. +//////////////////////////////////////////////////////////////////// +INLINE streampos Multifile:: +normalize_streampos(streampos fpos) const { + return word_to_streampos(streampos_to_word(fpos)); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE Multifile::Subfile:: +Subfile(const string &name) : + _name(name) +{ + _index_start = 0; + _data_start = 0; + _data_length = 0; + _source = (istream *)NULL; + _flags = 0; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::operator < +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE bool Multifile::Subfile:: +operator < (const Multifile::Subfile &other) const { + return _name < other._name; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::is_deleted +// Access: Public +// Description: Returns true if the Subfile indicates it has been +// deleted (removed from the index), false otherwise. +// This should never be true of Subfiles that currently +// appear in either the _subfiles or _new_subfiles +// lists. +//////////////////////////////////////////////////////////////////// +INLINE bool Multifile::Subfile:: +is_deleted() const { + return (_flags & SF_deleted) != 0; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::is_index_invalid +// Access: Public +// Description: Returns true if there was some problem reading the +// index record for this Subfile from the Multifile. +//////////////////////////////////////////////////////////////////// +INLINE bool Multifile::Subfile:: +is_index_invalid() const { + return (_flags & SF_index_invalid) != 0; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::is_data_invalid +// Access: Public +// Description: Returns true if there was some problem reading the +// data contents of this Subfile, particularly when +// copying into the Multifile. +//////////////////////////////////////////////////////////////////// +INLINE bool Multifile::Subfile:: +is_data_invalid() const { + return (_flags & SF_data_invalid) != 0; } diff --git a/panda/src/express/multifile.cxx b/panda/src/express/multifile.cxx index 00498f3997..1bdb088830 100644 --- a/panda/src/express/multifile.cxx +++ b/panda/src/express/multifile.cxx @@ -16,696 +16,1141 @@ // //////////////////////////////////////////////////////////////////// -#include -#include "config_express.h" -#include "error_utils.h" - -#include -#include - #include "multifile.h" -//////////////////////////////////////////////////////////////////// -// Defines -//////////////////////////////////////////////////////////////////// -PN_uint32 Multifile::_magic_number = 0xbeeffeeb; +#include "config_express.h" +#include -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::Constructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Multifile::Memfile:: -Memfile(void) { - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile::Memfile constructor called" << endl; - reset(); - _header_length_buf_length = sizeof(_header_length); - _header_length_buf = new char[_header_length_buf_length]; -} +// This sequence of bytes begins each Multifile to identify it as a +// Multifile. +const char Multifile::_header[] = "pmf\0\n\r"; +const size_t Multifile::_header_size = 6; -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::Destructor -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -Multifile::Memfile:: -~Memfile(void) { - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile::Memfile destructor called" << endl; - if (_buffer != (char *)0L) - delete _buffer; - _buffer = (char *)0L; - delete _header_length_buf; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::reset -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -void Multifile::Memfile:: -reset(void) { - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile::Memfile reset called" << endl; - _header_length_parsed = false; - _header_parsed = false; - _header_length = 0; - _buffer_length = 0; - //if (_buffer != (char *)0L) - // delete _buffer; - _buffer = (char *)0L; - _file_open = false; - _bytes_written = 0; - //_datagram.clear(); -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::parse_header_length -// Access: Public -// Description: Fills up _datagram until it has sizeof(_header_length) -// bytes and then extracts _header_length from _datagram. -// Returns true when this has been accomplished. -// Advances start to end of header_length. -//////////////////////////////////////////////////////////////////// -bool Multifile::Memfile:: -parse_header_length(char *&start, int &size) { - if (_header_length_parsed == true) - return true; - - int bytes_so_far = _datagram.get_length(); - if (bytes_so_far + size < _header_length_buf_length) { - _datagram.append_data(start, size); - start += size; - size = 0; - return false; - } - - _datagram.append_data(start, _header_length_buf_length - bytes_so_far); - nassertr((int)_datagram.get_length() == _header_length_buf_length, false); - - // Advance start and adjust size - nassertr(_header_length_buf_length >= bytes_so_far, false); - start += (_header_length_buf_length - bytes_so_far); - nassertr(size >= _header_length_buf_length, false); - size -= (_header_length_buf_length - bytes_so_far); - - DatagramIterator di(_datagram); - _header_length = di.get_int32(); - - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile::Memfile::parse_header_length() - header length: " - << _header_length << endl; - - nassertr(_header_length > _header_length_buf_length + (int)sizeof(_buffer_length), false); - - _header_length_parsed = true; - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::parse_header -// Access: Public -// Description: Returns true when a complete header has been parsed. -// Advances the start pointer to the end of the header. -//////////////////////////////////////////////////////////////////// -bool Multifile::Memfile:: -parse_header(char *&start, int& size) { - if (_header_parsed == true) - return true; - - // Determine header length - if (parse_header_length(start, size) == false) - return false; - - int bytes_so_far = _datagram.get_length(); - - // Make sure we don't exceed the length of the header - int tsize = size; - if ((size + bytes_so_far) >= _header_length) { - nassertr(bytes_so_far <= _header_length, false); - tsize = _header_length - bytes_so_far; - _header_parsed = true; - } - - // Accumulate bytes collected so far - _datagram.append_data(start, tsize); - - // Fill in data for the memfile - if (_header_parsed == true) { - nassertr((int)_datagram.get_length() == _header_length, false); - DatagramIterator di(_datagram); - PN_int32 header_length = di.get_int32(); - nassertr(header_length == _header_length, false); - _name = di.extract_bytes(_header_length - _header_length_buf_length - - sizeof(_buffer_length)); - _buffer_length = di.get_int32(); - nassertr(_buffer_length >= 0, false); - - express_cat.debug() - << "Multifile::Memfile::parse_header() - Got a header for mem " - << "file: " << _name << " header length: " << _header_length - << " buffer length: " << _buffer_length << endl; - - // Advance start pointer to the end of the header - start += tsize; - nassertr(size >= tsize, false); - size -= tsize; - _datagram.clear(); - } - - return _header_parsed; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::read -// Access: Public -// Description: Reads from an individual file -//////////////////////////////////////////////////////////////////// -bool Multifile::Memfile:: -read(const Filename &name) { - ifstream read_stream; - if (!name.open_read(read_stream)) { - express_cat.error() - << "Multifile::Memfile() - Failed to open input file: " - << name << endl; - return false; - } - - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile::Memfile::read() - Reading file: " << name << endl; - - _header_length = name.length() + sizeof(_header_length) + - sizeof(_buffer_length); - _name = name; - - // Determine the length of the file - read_stream.seekg(0, ios::end); - _buffer_length = read_stream.tellg(); - _buffer = new char[_buffer_length]; - - // Read the file - read_stream.seekg(0, ios::beg); - read_stream.read(_buffer, _buffer_length); - - return (_buffer_length > 0); -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::read_from_multifile -// Access: Public -// Description: Reads a Memfile from an open Multifile ifstream -//////////////////////////////////////////////////////////////////// -bool Multifile::Memfile:: -read_from_multifile(ifstream &read_stream) { - read_stream.read(_header_length_buf, _header_length_buf_length); - char *start = _header_length_buf; - int size = _header_length_buf_length; - if (parse_header_length(start, size) == false) { - express_cat.error() - << "Multifile::Memfile::read_from_multifile() - invalid header " - << "length" << endl; - return false; - } - - // Read the rest of the header - char *header_buf = new char[_header_length - _header_length_buf_length]; - read_stream.read(header_buf, _header_length - _header_length_buf_length); - start = header_buf; - size = _header_length; - if (parse_header(start, size) == false) { - delete header_buf; - express_cat.error() - << "Multifile::Memfile::read_from_multifile() - Invalid header" - << endl; - return false; - } - - delete header_buf; - - express_cat.debug() - << "Multifile::Memfile::read_from_multifile() - Got a valid Memfile " - << "header: name: " << _name << " length: " << _buffer_length << endl; - - _buffer = new char[_buffer_length]; - read_stream.read(_buffer, _buffer_length); - - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::write -// Access: Public -// Description: Writes to a individual file -//////////////////////////////////////////////////////////////////// -bool Multifile::Memfile:: -write(const Filename &rel_path) { - ofstream write_stream; - Filename name = rel_path.get_fullpath() + _name.get_fullpath(); - name.set_binary(); - name.make_dir(); - if (!name.open_write(write_stream)) { - express_cat.error() - << "Multifile::Memfile::write() - Failed to open output file: " - << name << endl; - return false; - } - - express_cat.debug() - << "Writing to file: " << name << endl; - - write_stream.write(_buffer, _buffer_length); - return true; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::write -// Access: Public -// Description: Writes a Memfile to an open Multifile ofstream -//////////////////////////////////////////////////////////////////// -void Multifile::Memfile:: -write_to_multifile(ofstream &write_stream) { - express_cat.debug() - << "Writing: " << _name << " to multifile" << endl; - - _datagram.clear(); - _datagram.add_int32(_header_length); - string path_name = _name.get_fullpath(); - _datagram.append_data(path_name.c_str(), path_name.length()); - _datagram.add_int32(_buffer_length); - - string msg = _datagram.get_message(); - write_stream.write((char *)msg.data(), msg.length()); - write_stream.write(_buffer, _buffer_length); -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::Memfile::write -// Access: Public -// Description: Returns true when the memfile has been parsed and -// written to disk. -// Advances the start pointer as it writes. -//////////////////////////////////////////////////////////////////// -int Multifile::Memfile:: -write(char *&start, int &size, const Filename &rel_path) { - // Make sure we've got a complete header first - if (parse_header(start, size) == false) { - return EU_ok; - } - - // Try to open the file for writing - if (_file_open == false) { - Filename name = rel_path.get_fullpath() + _name.get_fullpath(); - name.set_binary(); - name.make_dir(); - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile::Memfile::write() - Opening mem file: " << name - << " for writing" << endl; - if ((_file_open = name.open_write(_write_stream)) == false) { - express_cat.error() - << "Multfile::Memfile::write() - Couldn't open file: " - << name << " : " << strerror(errno) << endl; - return get_write_error(); - } - _file_open = true; - } - - // Don't write more than the buffer length - int done = EU_ok; - int tsize = size; - nassertr(_buffer_length >= _bytes_written, false); - int missing_bytes = _buffer_length - _bytes_written; - if (size >= missing_bytes) { - tsize = missing_bytes; - done = EU_success; - } - - _write_stream.write(start, tsize); - start += tsize; - _bytes_written += tsize; - nassertr(size >= tsize, false); - size -= tsize; - - if (done == EU_success) { - _write_stream.close(); - express_cat.debug() - << "Multifile::Memfile::write() - Closing mem file" << endl; - } - - return done; -} +// These numbers identify the version of the Multifile. Generally, a +// change in the major version is intolerable; while a Multifile with +// an older minor version may still be read. +const int Multifile::_current_major_ver = 1; +const int Multifile::_current_minor_ver = 0; +// +// A Multifile consists of the following elements: +// +// (1) A header. This is always the first n bytes of the Multifile, +// and contains a magic number to identify the file, as well as +// version numbers and any file-specific parameters. +// +// char[6] The string Multifile::_header, a magic number. +// int16 The file's major version number +// int16 The file's minor version number +// uint32 Scale factor. This scales all address references within +// the file. Normally 1, this may be set larger to +// support Multifiles larger than 4GB. +// +// (2) Zero or more index entries, one for each subfile within the +// Multifile. These entries are of variable length. The first one of +// these immediately follows the header, and the first word of each +// index entry contains the address of the next index entry. A zero +// "next" address marks the end of the chain. These may appear at any +// point within the Multifile; they do not necessarily appear in +// sequential order at the beginning of the file (although they will +// after the file has been "packed"). +// +// uint32 The address of the next entry. +// uint32 The address of this subfile's data record. +// uint32 The length in bytes of this subfile's data record. +// uint16 The Subfile::_flags member. +// uint16 The length in bytes of the subfile's name. +// char[n] The subfile's name. +// +// (3) Zero or more data entries, one for each subfile. These may +// appear at any point within the Multifile; they do not necessarily +// follow each index entry, nor are they necessarily all grouped +// together at the end (although they will be all grouped together at +// the end after the file has been "packed"). These are just blocks +// of literal data. +// //////////////////////////////////////////////////////////////////// // Function: Multifile::Constructor -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// Multifile:: -Multifile(void) { - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile constructor called" << endl; - reset(); - _header_length = sizeof(_magic_number) + sizeof(_num_mfiles); +Multifile() { + _read = (istream *)NULL; + _write = (ostream *)NULL; + _next_index = 0; + _last_index = 0; + _needs_repack = false; + _scale_factor = 1; + _file_major_ver = 0; + _file_minor_ver = 0; + _open_subfile = (Subfile *)NULL; } //////////////////////////////////////////////////////////////////// // Function: Multifile::Destructor -// Access: Public +// Access: Published // Description: //////////////////////////////////////////////////////////////////// Multifile:: -~Multifile(void) { - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile destructor called" << endl; - _files.erase(_files.begin(), _files.end()); - if (_current_mfile != NULL) - delete _current_mfile; +~Multifile() { + close(); } //////////////////////////////////////////////////////////////////// -// Function: Multifile::evaluate -// Access: Public -// Description: Checks for a valid compressed or uncompressed +// Function: Multifile::open_read +// Access: Published +// Description: Opens the named Multifile on disk for reading. The +// Multifile index is read in, and the list of subfiles +// becomes available; individual subfiles may then be +// extracted or read, but the list of subfiles may not +// be modified. +// +// Also see the version of open_read() which accepts an +// istream. Returns true on success, false on failure. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +open_read(const Filename &multifile_name) { + close(); + Filename fname = multifile_name; + fname.set_binary(); + if (!fname.open_read(_read_file)) { + return false; + } + _read = &_read_file; + _multifile_name = multifile_name; + return read_index(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::open_write +// Access: Published +// Description: Opens the named Multifile on disk for writing. If +// there already exists a file by that name, it is +// deleted. The Multifile is then prepared for +// accepting a brand new set of subfiles, which will be +// written to the indicated filename. Individual +// subfiles may not be extracted or read. +// +// Also see the version of open_write() which accepts an +// ostream. Returns true on success, false on failure. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +open_write(const Filename &multifile_name) { + close(); + Filename fname = multifile_name; + fname.set_binary(); + if (!fname.open_write(_write_file)) { + return false; + } + _write = &_write_file; + _multifile_name = multifile_name; + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::open_read_write +// Access: Published +// Description: Opens the named Multifile on disk for reading and +// writing. If there already exists a file by that +// name, its index is read. Subfiles may be added or +// removed, and the resulting changes will be written to +// the named file. +// +// Also see the version of open_read_write() which +// accepts an iostream. Returns true on success, false +// on failure. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +open_read_write(const Filename &multifile_name) { + close(); + Filename fname = multifile_name; + fname.set_binary(); + bool exists = fname.exists(); + if (!fname.open_read_write(_read_write_file)) { + return false; + } + _read = &_read_write_file; + _write = &_read_write_file; + _multifile_name = multifile_name; + + if (exists) { + return read_index(); + } else { + return true; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::close +// Access: Published +// Description: Closes the Multifile if it is open. All changes are +// flushed to disk, and the file becomes invalid for +// further operations until the next call to open(). +//////////////////////////////////////////////////////////////////// +void Multifile:: +close() { + close_subfile(); + flush(); + _read = (istream *)NULL; + _write = (ostream *)NULL; + _next_index = 0; + _last_index = 0; + _needs_repack = false; + _scale_factor = 1; + _file_major_ver = 0; + _file_minor_ver = 0; + + _read_file.close(); + _write_file.close(); + _read_write_file.close(); + _multifile_name = Filename(); + + clear_subfiles(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::add_subfile +// Access: Published +// Description: Adds a file on disk as a subfile to the Multifile. +// The file named by filename will be read and added to +// the Multifile at the next call to flush(). +// +// Returns true on success, false on failure. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +add_subfile(const string &subfile_name, const Filename &filename) { + nassertr(is_write_valid(), false); + + if (!filename.exists()) { + return false; + } + Subfile *subfile = new Subfile(subfile_name); + + subfile->_source_filename = filename; + subfile->_source_filename.set_binary(); + + return add_new_subfile(subfile); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::flush +// Access: Published +// Description: Writes all contents of the Multifile to disk. Until +// flush() is called, add_subfile() and remove_subfile() +// do not actually do anything to disk. At this point, +// all of the recently-added subfiles are read and their +// contents are added to the end of the Multifile, and +// the recently-removed subfiles are marked gone from +// the Multifile. +// +// This may result in a suboptimal index. To guarantee +// that the index is written at the beginning of the +// file, call repack() instead of flush(). +// +// It is not necessary to call flush() explicitly unless +// you are concerned about reading the recently-added +// subfiles immediately. +// +// Returns true on success, false on failure. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +flush() { + if (!is_write_valid()) { + return false; + } + + if (_next_index == 0) { + // If we don't have an index yet, we don't have a header. Write + // the header. + if (!write_header()) { + return false; + } + } + + nassertr(_write != (ostream *)NULL, false); + + // First, mark out all of the removed subfiles. + PendingSubfiles::iterator pi; + for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { + Subfile *subfile = (*pi); + subfile->rewrite_index_flags(*_write); + delete subfile; + } + _removed_subfiles.clear(); + + bool wrote_ok = true; + + if (!_new_subfiles.empty()) { + // Add a few more files to the end. We always add subfiles at the + // end of the multifile, so go there first. + if (_last_index != 0) { + _write->seekp(0, ios::end); + if (_write->fail()) { + express_cat.info() + << "Unable to seek Multifile " << _multifile_name << ".\n"; + return false; + } + _next_index = _write->tellp(); + _next_index = pad_to_streampos(_next_index); + + // And update the forward link from the last_index to point to + // this new index location. + _write->seekp(_last_index); + Datagram dg; + dg.add_uint32(streampos_to_word(_next_index)); + _write->write(dg.get_data(), dg.get_length()); + } + + _write->seekp(_next_index); + nassertr(_next_index == _write->tellp(), false); + + // Ok, here we are at the end of the file. Write out the + // recently-added subfiles here. First, count up the index size. + for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { + Subfile *subfile = (*pi); + _last_index = _next_index; + _next_index = subfile->write_index(*_write, _next_index, this); + _next_index = pad_to_streampos(_next_index); + nassertr(_next_index == _write->tellp(), false); + } + + // Now we're at the end of the index. Write a 0 here to mark the + // end. + Datagram dg; + dg.add_uint32(0); + _write->write(dg.get_data(), dg.get_length()); + _next_index += 4; + _next_index = pad_to_streampos(_next_index); + + // All right, now write out each subfile's data. + for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { + Subfile *subfile = (*pi); + _next_index = subfile->write_data(*_write, _read, _next_index); + _next_index = pad_to_streampos(_next_index); + if (subfile->is_data_invalid()) { + wrote_ok = false; + } + nassertr(_next_index == _write->tellp(), false); + } + + // Now go back and fill in the proper addresses for the data start. + // We didn't do it in the first pass, because we don't really want + // to keep all those pile handles open, and so we didn't have to + // determine each pile's length ahead of time. + for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) { + Subfile *subfile = (*pi); + subfile->rewrite_index_data_start(*_write, this); + } + + _new_subfiles.clear(); + } + + _write->flush(); + if (!wrote_ok || _write->fail()) { + express_cat.info() + << "Unable to update Multifile " << _multifile_name << ".\n"; + close(); + return false; + } + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::repack +// Access: Published +// Description: Forces a complete rewrite of the Multifile and all of +// its contents, so that its index will appear at the +// beginning of the file with all of the subfiles listed +// in alphabetical order. This is considered optimal +// for reading, and is the standard configuration; but +// it is not essential to do this. +// +// It is only valid to call this if the Multifile was +// opened using open_read_write() and an explicit +// filename, rather than an iostream. Also, we must +// have write permission to the directory containing the +// Multifile. +// +// Returns true on success, false on failure. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +repack() { + nassertr(is_write_valid() && is_read_valid(), false); + nassertr(!_multifile_name.empty(), false); + + // First, we open a temporary filename to copy the Multifile to. + Filename temp_filename = + Filename::temporary(_multifile_name.get_dirname(), "mftemp"); + temp_filename.set_binary(); + ofstream temp; + if (!temp_filename.open_write(temp)) { + express_cat.info() + << "Unable to open temporary file " << temp_filename << "\n"; + return false; + } + + // Now we scrub our internal structures so it looks like we're a + // brand new Multifile. + PendingSubfiles::iterator pi; + for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { + Subfile *subfile = (*pi); + delete subfile; + } + _removed_subfiles.clear(); + _new_subfiles.clear(); + copy(_subfiles.begin(), _subfiles.end(), back_inserter(_new_subfiles)); + _next_index = 0; + _last_index = 0; + + // And we write our contents to our new temporary file. + _write = &temp; + if (!flush()) { + temp.close(); + temp_filename.unlink(); + return false; + } + + // Now close everything, and move the temporary file back over our + // original file. + Filename orig_name = _multifile_name; + temp.close(); + close(); + if (!temp_filename.rename_to(orig_name)) { + express_cat.info() + << "Unable to rename temporary file " << temp_filename << " to " + << orig_name.get_basename() << ".\n"; + return false; + } + + if (!open_read_write(orig_name)) { + express_cat.info() + << "Unable to read newly repacked " << _multifile_name + << ".\n"; + return false; + } + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::get_num_subfiles +// Access: Published +// Description: Returns the number of subfiles within the Multifile. +// The subfiles may be accessed in alphabetical order by +// iterating through [0 .. get_num_subfiles()). +//////////////////////////////////////////////////////////////////// +int Multifile:: +get_num_subfiles() const { + return _subfiles.size(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::find_subfile +// Access: Published +// Description: Returns the index of the subfile with the indicated +// name, or -1 if the named subfile is not within the // Multifile. //////////////////////////////////////////////////////////////////// int Multifile:: -evaluate(const char *start, int size) { - if (size < (int)sizeof(_magic_number)) { - express_cat.debug() - << "Multifile::evaluate() - not enough bytes to determine" << endl; - return T_unknown; +find_subfile(const string &subfile_name) const { + Subfile find_subfile(subfile_name); + Subfiles::const_iterator fi; + fi = _subfiles.find(&find_subfile); + if (fi == _subfiles.end()) { + // Not present. + return -1; } - - Datagram dgram; - dgram.append_data(start, sizeof(_magic_number)); - DatagramIterator di(dgram); - PN_uint32 magic_number = di.get_uint32(); - if (magic_number == _magic_number) - return T_valid; - - express_cat.debug() - << "Invalid magic number: " << magic_number << " (" - << _magic_number << ")" << endl; - - return T_invalid; + return (fi - _subfiles.begin()); } //////////////////////////////////////////////////////////////////// -// Function: Multifile::parse_header -// Access: Public -// Description: Returns true when a complete header has been parsed. +// Function: Multifile::remove_subfile +// Access: Published +// Description: Removes the nth subfile from the Multifile. This +// will cause all subsequent index numbers to decrease +// by one. The file will not actually be removed from +// the disk until the next call to flush(). +// +// Note that this does not actually remove the data from +// the indicated subfile; it simply removes it from the +// index. The Multifile will not be reduced in size +// after this operation, until the next call to +// repack(). //////////////////////////////////////////////////////////////////// -int Multifile:: -parse_header(char *&start, int &size) { - if (_header_parsed == true) - return EU_success; +void Multifile:: +remove_subfile(int index) { + nassertv(is_write_valid()); + nassertv(index >= 0 && index < (int)_subfiles.size()); + Subfile *subfile = _subfiles[index]; + subfile->_flags |= SF_deleted; + _removed_subfiles.push_back(subfile); + _subfiles.erase(_subfiles.begin() + index); - int dgramsize = _datagram.get_length(); - int tsize = size; - - // Make sure we don't exceed the length of the header - nassertr(_header_length >= dgramsize, EU_error_abort); - int missing_bytes = _header_length - dgramsize; - if (size >= missing_bytes) { - tsize = missing_bytes; - _header_parsed = true; - } - - // Accumulate bytes collected so far - _datagram.append_data(start, tsize); - - // Verify magic number - if (_header_parsed == true) { - DatagramIterator di(_datagram); - PN_uint32 magic_number = di.get_uint32(); - if (magic_number != _magic_number) { - express_cat.error() - << "Multifile::parse_header() - Invalid magic number: " - << magic_number << " (" << _magic_number << ")" << endl; - return EU_error_abort; - } - _num_mfiles = di.get_int32(); - if (_num_mfiles <= 0) { - express_cat.debug() - << "Multifile::parse_header() - No memfiles in multifile" - << endl; - return EU_error_file_empty; - } - - // Advance start pointer to the end of the header - start += tsize; - nassertr(size >= tsize, false); - size -= tsize; - _datagram.clear(); - return EU_success; - } - - return EU_ok; + _needs_repack = true; } //////////////////////////////////////////////////////////////////// -// Function: Multifile::add -// Access: Public -// Description: +// Function: Multifile::get_subfile_name +// Access: Published +// Description: Returns the name of the nth subfile. +//////////////////////////////////////////////////////////////////// +const string &Multifile:: +get_subfile_name(int index) const { +#ifndef NDEBUG + static string empty_string; + nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string); +#endif + return _subfiles[index]->_name; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::get_subfile_length +// Access: Published +// Description: Returns the data length of the nth subfile. This +// might return 0 if the subfile has recently been added +// and flush() has not yet been called. +//////////////////////////////////////////////////////////////////// +size_t Multifile:: +get_subfile_length(int index) const { + nassertr(index >= 0 && index < (int)_subfiles.size(), 0); + return _subfiles[index]->_data_length; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::read_subfile +// Access: Published +// Description: Fills the indicated Datagram with the data from the +// nth subfile. +//////////////////////////////////////////////////////////////////// +void Multifile:: +read_subfile(int index, Datagram &data) { + nassertv(is_read_valid()); + nassertv(index >= 0 && index < (int)_subfiles.size()); + data.clear(); + + istream &in = open_read_subfile(index); + size_t length = _subfiles[index]->_data_length; + for (size_t p = 0; p < length; p++) { + int byte = in.get(); + data.add_int8(byte); + } + bool failed = in.fail(); + close_subfile(); + nassertv(!failed); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::extract_subfile +// Access: Published +// Description: Extracts the nth subfile into a file with the given +// name. //////////////////////////////////////////////////////////////////// bool Multifile:: -add(const Filename &name) { - Memfile *mfile = new Memfile; - if (mfile->read(name)) { - _files.push_back(mfile); - return true; - } - return false; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::remove -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -bool Multifile:: -remove(const Filename &name) { - MemfileList::iterator found; - found = find_if(_files.begin(), _files.end(), MemfileMatch(name)); - if (found != _files.end()) { - _files.erase(found); - return true; - } - return false; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::has_file -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -bool Multifile:: -has_file(const Filename &name) { - MemfileList::iterator found; - found = find_if(_files.begin(), _files.end(), MemfileMatch(name)); - return (found != _files.end()); -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::read -// Access: Public -// Description: Reads a multifile from disk -//////////////////////////////////////////////////////////////////// -bool Multifile:: -read(Filename &name) { - - // Open the multifile for reading - ifstream read_stream; - name.set_binary(); - if (!name.open_read(read_stream)) { - express_cat.error() - << "Multifile::read() - Failed to open input file: " - << name << endl; +extract_subfile(int index, const Filename &filename) { + nassertr(is_read_valid(), false); + nassertr(index >= 0 && index < (int)_subfiles.size(), false); + + Filename fname = filename; + fname.set_binary(); + fname.make_dir(); + ofstream out; + if (!fname.open_write(out)) { + express_cat.info() + << "Unable to write to file " << filename << "\n"; return false; } - // Check for a valid header - char *buffer = new char[_header_length]; - read_stream.read(buffer, _header_length); - char *start = buffer; - int len = _header_length; - int ret = parse_header(start, len); - delete buffer; - if (ret < 0) { - express_cat.error() - << "Multifile::read() - invalid header" << endl; - return false; - } + return extract_subfile_to(index, out); +} - if (_num_mfiles <= 0) { - express_cat.error() - << "Multfile::read() - No files in Multfile: " << name << endl; - return false; - } - - // Read all the Memfiles - for (int i = 0; i < _num_mfiles; i++) { - Memfile *mfile = new Memfile; - mfile->read_from_multifile(read_stream); - _files.push_back(mfile); - } +//////////////////////////////////////////////////////////////////// +// Function: Multifile::open_read +// Access: Public +// Description: Opens an anonymous Multifile for reading using an +// istream. There must be seek functionality via +// seekg() and tellg() on the istream. +// +// This version of open_read() does not close the +// istream when Multifile.close() is called. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +open_read(istream *multifile_stream) { + close(); + _read = multifile_stream; + return read_index(); +} +//////////////////////////////////////////////////////////////////// +// Function: Multifile::open_write +// Access: Public +// Description: Opens an anonymous Multifile for writing using an +// ostream. There must be seek functionality via +// seekp() and tellp() on the pstream. +// +// This version of open_write() does not close the +// ostream when Multifile.close() is called. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +open_write(ostream *multifile_stream) { + close(); + _write = multifile_stream; return true; } //////////////////////////////////////////////////////////////////// -// Function: Multifile::write +// Function: Multifile::open_read_write // Access: Public -// Description: +// Description: Opens an anonymous Multifile for reading and writing +// using an iostream. There must be seek functionality +// via seekg()/seekp() and tellg()/tellp() on the +// iostream. +// +// This version of open_read_write() does not close the +// iostream when Multifile.close() is called. //////////////////////////////////////////////////////////////////// bool Multifile:: -write(Filename name) { +open_read_write(iostream *multifile_stream) { + close(); + _read = multifile_stream; + _write = multifile_stream; - ofstream write_stream; - name.set_binary(); - if (!name.open_write(write_stream)) { - express_cat.error() - << "Multifile::write() - Error opening file: " << name << endl; - return false; + // Check whether the read stream is empty. + _read->seekg(0, ios::end); + if (_read->tellg() == 0) { + // The read stream is empty, which is always valid. + return true; } - _num_mfiles = _files.size(); - if (_num_mfiles == 0) { - express_cat.debug() - << "No files in Multifile to write" << endl; - return false; + // The read stream is not empty, so we'd better have a valid + // Multifile. + _read->seekg(0); + return read_index(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::add_subfile +// Access: Published +// Description: Adds a file on disk as a subfile to the Multifile. +// The indicated istream will be read and its contents +// added to the Multifile at the next call to flush(). +//////////////////////////////////////////////////////////////////// +bool Multifile:: +add_subfile(const string &subfile_name, istream *subfile_data) { + nassertr(is_write_valid(), false); + + Subfile *subfile = new Subfile(subfile_name); + + subfile->_source = subfile_data; + + return add_new_subfile(subfile); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::extract_subfile_to +// Access: Public +// Description: Extracts the nth subfile to the indicated ostream. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +extract_subfile_to(int index, ostream &out) { + nassertr(is_read_valid(), false); + nassertr(index >= 0 && index < (int)_subfiles.size(), false); + + istream &in = open_read_subfile(index); + size_t length = _subfiles[index]->_data_length; + for (size_t p = 0; p < length; p++) { + int byte = in.get(); + out.put(byte); + } + bool failed = in.fail(); + close_subfile(); + nassertr(!failed, false); + + return (!out.fail()); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::open_read_subfile +// Access: Public +// Description: Returns an istream that may be used to read the +// indicated subfile. The istream may or may not be a +// reference within the Multifile itself, so relative +// seeks are acceptable but global seeks will fail. +// Also, checking for eof() may fail; you must use +// get_subfile_length() instead. +// +// It is not valid to perform any other operations on +// this Multifile until the indicated subfile has been +// read to completion and close_subfile() has been +// called. +//////////////////////////////////////////////////////////////////// +istream &Multifile:: +open_read_subfile(int index) { +#ifndef NDEBUG + ifstream empty_stream; + nassertr(_open_subfile == (Subfile *)NULL, empty_stream); + nassertr(is_read_valid(), empty_stream); + nassertr(index >= 0 && index < (int)_subfiles.size(), empty_stream); +#endif + _open_subfile = _subfiles[index]; + if (_open_subfile->_source != (istream *)NULL) { + _open_subfile->_source->seekg(0); + return *_open_subfile->_source; + } + if (!_open_subfile->_source_filename.empty()) { + _open_subfile->_source_filename.open_read(_subfile_read); + return _subfile_read; + } + nassertr(_open_subfile->_data_start != 0, empty_stream); + _read->seekg(_open_subfile->_data_start); + return *_read; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::close_subfile +// Access: Public +// Description: "Closes" the istream that was returned via a previous +// call to open_read_subfile(), and makes other +// operations on the Multifile valid once more. +//////////////////////////////////////////////////////////////////// +void Multifile:: +close_subfile() { + _open_subfile = (Subfile *)NULL; + _subfile_read.close(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::pad_to_streampos +// Access: Private +// Description: Assumes the _write pointer is at the indicated fpos, +// rounds the fpos up to the next legitimate address +// (using normalize_streampos()), and writes enough +// zeros to the stream to fill the gap. Returns the new +// fpos. +//////////////////////////////////////////////////////////////////// +streampos Multifile:: +pad_to_streampos(streampos fpos) { + nassertr(_write != (ostream *)NULL, fpos); + nassertr(_write->tellp() == fpos, fpos); + streampos new_fpos = normalize_streampos(fpos); + while (fpos < new_fpos) { + _write->put(0); + fpos++; + } + return fpos; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::add_new_subfile +// Access: Private +// Description: Adds a newly-allocated Subfile pointer to the +// Multifile. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +add_new_subfile(Subfile *subfile) { + if (_next_index != 0) { + // If we're adding a Subfile to an already-existing Multifile, we + // will eventually need to repack the file. + _needs_repack = true; } - express_cat.debug() - << "Multifile::write() - Writing multifile: " << name << endl; - - write_header(write_stream); - - MemfileList::iterator i; - for (i = _files.begin(); i != _files.end(); ++i) - (*i)->write_to_multifile(write_stream); - - write_stream.close(); + pair insert_result = _subfiles.insert(subfile); + if (!insert_result.second) { + // Hmm, unable to insert. There must already be a subfile by that + // name. Remove the old one. + Subfile *old_subfile = (*insert_result.first); + old_subfile->_flags |= SF_deleted; + _removed_subfiles.push_back(old_subfile); + (*insert_result.first) = subfile; + } + _new_subfiles.push_back(subfile); return true; } //////////////////////////////////////////////////////////////////// -// Function: Multifile::write -// Access: Public -// Description: Returns true when all the memfiles have been -// written. -// Advances the start pointer as it writes. +// Function: Multifile::clear_subfiles +// Access: Private +// Description: Removes the set of subfiles from the tables and frees +// their associated memory. //////////////////////////////////////////////////////////////////// -int Multifile:: -write(char *&start, int &size, const Filename &rel_path) { - // Make sure we have a complete header first - int parse_ret = parse_header(start, size); - if (parse_ret < 0) { - express_cat.error() - << "Multifile::write() - bad header" << endl; - return parse_ret; +void Multifile:: +clear_subfiles() { + PendingSubfiles::iterator pi; + for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) { + Subfile *subfile = (*pi); + subfile->rewrite_index_flags(*_write); + delete subfile; + } + _removed_subfiles.clear(); + + // We don't have to delete the ones in _new_subfiles, because these + // also appear in _subfiles. + _new_subfiles.clear(); + + Subfiles::iterator fi; + for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) { + Subfile *subfile = (*fi); + delete subfile; + } + _subfiles.clear(); +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::read_index +// Access: Private +// Description: Reads the Multifile header and index. Returns true +// if successful, false if the Multifile is not valid. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +read_index() { + nassertr(_read != (istream *)NULL, false); + static const size_t header_followup_size = 2 + 2 + 4; + static const size_t total_header_size = _header_size + header_followup_size; + char this_header[total_header_size]; + _read->read(this_header, total_header_size); + if (_read->fail() || _read->gcount() != total_header_size) { + express_cat.info() + << "Unable to read Multifile header " << _multifile_name << ".\n"; + close(); + return false; + } + if (memcmp(this_header, _header, _header_size) != 0) { + express_cat.info() + << _multifile_name << " is not a Multifile.\n"; + return false; } - while (_num_mfiles > 0) { - if (_current_mfile == NULL) { - _current_mfile = new Memfile; - } - int write_ret = _current_mfile->write(start, size, rel_path); - if (write_ret == EU_success) { - _num_mfiles--; - delete _current_mfile; - _current_mfile = NULL; + // Now get the version numbers out. + Datagram dg(this_header + _header_size, header_followup_size); + DatagramIterator dgi(dg); + _file_major_ver = dgi.get_int16(); + _file_minor_ver = dgi.get_int16(); + _scale_factor = dgi.get_uint32(); + + if (_file_major_ver != _current_major_ver || + (_file_major_ver == _current_major_ver && + _file_minor_ver > _current_minor_ver)) { + express_cat.info() + << _multifile_name << " has version " << _file_major_ver << "." + << _file_minor_ver << ", expecting version " + << _current_major_ver << "." << _current_minor_ver << ".\n"; + return false; + } + + + // Now read the index out. + _next_index = _read->tellg(); + _next_index = normalize_streampos(_next_index); + _read->seekg(_next_index); + _last_index = 0; + streampos index_forward; + + Subfile *subfile = new Subfile(""); + index_forward = subfile->read_index(*_read, _next_index, this); + while (index_forward != 0) { + _last_index = _next_index; + if (subfile->is_deleted()) { + // Ignore deleted Subfiles in the index. + _needs_repack = true; + delete subfile; } else { - if (write_ret < 0) { - express_cat.error() - << "Multifile::write() - bad write: " << write_ret << endl; - } - return write_ret; + _subfiles.push_back(subfile); + } + if (index_forward != normalize_streampos(_read->tellg())) { + // If the index entries don't follow exactly sequentially, the + // file ought to be repacked. + _needs_repack = true; + } + _read->seekg(index_forward); + _next_index = index_forward; + subfile = new Subfile(""); + index_forward = subfile->read_index(*_read, _next_index, this); + } + if (subfile->is_index_invalid()) { + express_cat.info() + << "Error reading index for " << _multifile_name << ".\n"; + close(); + delete subfile; + return false; + } + + size_t before_size = _subfiles.size(); + _subfiles.sort(); + size_t after_size = _subfiles.size(); + + // If these don't match, the same filename appeared twice in the + // index, which shouldn't be possible. + nassertr(before_size == after_size, true); + + delete subfile; + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::write_header +// Access: Private +// Description: Writes just the header part of the Multifile, not the +// index. +//////////////////////////////////////////////////////////////////// +bool Multifile:: +write_header() { + nassertr(_write != (ostream *)NULL, false); + nassertr(_write->tellp() == 0, false); + _write->write(_header, _header_size); + Datagram dg; + dg.add_int16(_current_major_ver); + dg.add_int16(_current_minor_ver); + dg.add_uint32(_scale_factor); + _write->write(dg.get_data(), dg.get_length()); + + _next_index = _write->tellp(); + _next_index = pad_to_streampos(_next_index); + _last_index = 0; + + if (_write->fail()) { + express_cat.info() + << "Unable to write header for " << _multifile_name << ".\n"; + close(); + return false; + } + + return true; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::read_index +// Access: Public +// Description: Reads the index record for the Subfile from the +// indicated istream. Assumes the istream has already +// been positioned to the indicated stream position, +// fpos, the start of the index record. Returns the +// position within the file of the next index record. +//////////////////////////////////////////////////////////////////// +streampos Multifile::Subfile:: +read_index(istream &read, streampos fpos, Multifile *multifile) { + nassertr(read.tellg() == fpos, fpos); + + // First, get the next stream position. We do this separately, + // because if it is zero, we don't get anything else. + + static const size_t next_index_size = 4; + char next_index_buffer[next_index_size]; + read.read(next_index_buffer, next_index_size); + if (read.fail() || read.gcount() != next_index_size) { + _flags |= SF_index_invalid; + return 0; + } + + Datagram idg(next_index_buffer, next_index_size); + DatagramIterator idgi(idg); + streampos next_index = multifile->word_to_streampos(idgi.get_uint32()); + + if (next_index == 0) { + return 0; + } + + // Now get the rest of the index (except the name, which is variable + // length). + + _index_start = fpos; + + static const size_t index_size = 4 + 4 + 2 + 2; + char index_buffer[index_size]; + read.read(index_buffer, index_size); + if (read.fail() || read.gcount() != index_size) { + _flags |= SF_index_invalid; + return 0; + } + + Datagram dg(index_buffer, index_size); + DatagramIterator dgi(dg); + _data_start = multifile->word_to_streampos(dgi.get_uint32()); + _data_length = dgi.get_uint32(); + _flags = dgi.get_uint16(); + size_t name_length = dgi.get_uint16(); + + // And finally, get the rest of the name. + char *name_buffer = new char[name_length]; + nassertr(name_buffer != (char *)NULL, next_index); + for (size_t ni = 0; ni < name_length; ni++) { + name_buffer[ni] = read.get() ^ 0xff; + } + _name = string(name_buffer, name_length); + delete[] name_buffer; + + return next_index; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::write_index +// Access: Public +// Description: Writes the index record for the Subfile to the +// indicated ostream. Assumes the istream has already +// been positioned to the indicated stream position, +// fpos, the start of the index record, and that this is +// the effective end of the file. Returns the position +// within the file of the next index record. +// +// The _index_start member is updated by this operation. +//////////////////////////////////////////////////////////////////// +streampos Multifile::Subfile:: +write_index(ostream &write, streampos fpos, Multifile *multifile) { + nassertr(write.tellp() == fpos, fpos); + + _index_start = fpos; + + // This will be the contents of this particular index record. + Datagram dg; + dg.add_uint32(multifile->streampos_to_word(_data_start)); + dg.add_uint32(_data_length); + dg.add_uint16(_flags); + dg.add_uint16(_name.length()); + + // For no real good reason, we'll invert all the bits in the name. + // The only reason we do this is to make it inconvenient for a + // casual browser of the Multifile to discover the names of the + // files stored within it. Naturally, this isn't real obfuscation + // or security. + string::iterator ni; + for (ni = _name.begin(); ni != _name.end(); ++ni) { + dg.add_int8((*ni) ^ 0xff); + } + + size_t this_index_size = 4 + dg.get_length(); + + // Plus, we will write out the next index address first. + streampos next_index = fpos + this_index_size; + Datagram idg; + idg.add_uint32(multifile->streampos_to_word(next_index)); + + write.write(idg.get_data(), idg.get_length()); + write.write(dg.get_data(), dg.get_length()); + + return next_index; +} + +//////////////////////////////////////////////////////////////////// +// Function: Multifile::Subfile::write_index +// Access: Public +// Description: Writes the data record for the Subfile to the +// indicated ostream: the actual contents of the +// Subfile. Assumes the istream has already been +// positioned to the indicated stream position, fpos, +// the start of the data record, and that this is the +// effective end of the file. Returns the position +// within the file of the next data record. +// +// The _data_start and _data_length members are updated +// by this operation. +// +// If the "read" pointer is non-NULL, it is the readable +// istream of a Multifile in which the Subfile might +// already be packed. This is used for reading the +// contents of the Subfile during a repack() operation. +//////////////////////////////////////////////////////////////////// +streampos Multifile::Subfile:: +write_data(ostream &write, istream *read, streampos fpos) { + nassertr(write.tellp() == fpos, fpos); + + _data_start = fpos; + + istream *source = _source; + ifstream source_file; + if (source == (istream *)NULL && !_source_filename.empty()) { + // If we have a filename, open it up and read that. + if (!_source_filename.open_read(source_file)) { + // Unable to open the source file. + express_cat.info() + << "Unable to read " << _source_filename << ".\n"; + _flags |= SF_data_invalid; + _data_length = 0; + } else { + source = &source_file; } } - return EU_success; -} - -//////////////////////////////////////////////////////////////////// -// Function: Multifile::write_extract -// Access: Public -// Description: Returns true when entire Multifile has been -// extracted to disk files. -//////////////////////////////////////////////////////////////////// -bool Multifile:: -write_extract(char *&start, int &size, const Filename &rel_path) { - int parse_ret = parse_header(start, size); - if (parse_ret < 0) - return false; - if (_current_mfile == (Memfile *)0L) - _current_mfile = new Memfile; - for (;;) { - if (_current_mfile->write(start, size, rel_path) == false) - return false; - if (++_mfiles_written == _num_mfiles) - return true; - _current_mfile->reset(); + if (source == (istream *)NULL) { + // We don't have any source data. Perhaps we're reading from an + // already-packed Subfile (e.g. during repack()). + if (read == (istream *)NULL) { + // No, we're just screwed. + express_cat.info() + << "No source for subfile " << _name << ".\n"; + _flags |= SF_data_invalid; + } else { + // Read the data from the original Multifile. + for (size_t p = 0; p < _data_length; p++) { + int byte = read->get(); + if (read->eof() || read->fail()) { + // Unexpected EOF or other failure on the source file. + express_cat.info() + << "Unexpected EOF for subfile " << _name << ".\n"; + _flags |= SF_data_invalid; + } + write.put(byte); + } + } + } else { + // We do have source data. Copy it in, and also measure its + // length. + _data_length = 0; + int byte = source->get(); + while (!source->eof() && !source->fail()) { + _data_length++; + write.put(byte); + byte = source->get(); + } } + + _source = (istream *)NULL; + _source_filename = Filename(); + source_file.close(); + + return fpos + _data_length; } //////////////////////////////////////////////////////////////////// -// Function: Multifile::reset +// Function: Multifile::Subfile::rewrite_index_data_start // Access: Public -// Description: +// Description: Seeks within the indicate fstream back to the index +// record and rewrites just the _data_start and +// _data_length part of the index record. //////////////////////////////////////////////////////////////////// -void Multifile:: -reset(void) { - if (express_cat.is_debug()) - express_cat.debug() - << "Multifile reset called" << endl; - _header_parsed = false; - _num_mfiles = 0; - _current_mfile = (Memfile *)0L; - _datagram.clear(); - _files.erase(_files.begin(), _files.end()); - _mfiles_written = 0; +void Multifile::Subfile:: +rewrite_index_data_start(ostream &write, Multifile *multifile) { + nassertv(_index_start != 0); + + static const size_t data_start_offset = 4; + size_t data_start_pos = _index_start + data_start_offset; + write.seekp(data_start_pos); + nassertv(!write.fail()); + + Datagram dg; + dg.add_uint32(multifile->streampos_to_word(_data_start)); + dg.add_uint32(_data_length); + write.write(dg.get_data(), dg.get_length()); } //////////////////////////////////////////////////////////////////// -// Function: Multifile::extract +// Function: Multifile::Subfile::rewrite_index_flags // Access: Public -// Description: +// Description: Seeks within the indicate fstream back to the index +// record and rewrites just the _flags part of the +// index record. //////////////////////////////////////////////////////////////////// -bool Multifile:: -extract(const Filename &name, const Filename &rel_path) { - MemfileList::iterator found; - found = find_if(_files.begin(), _files.end(), MemfileMatch(name)); - if (found != _files.end()) { - (*found)->write(rel_path); - return true; - } - return false; -} +void Multifile::Subfile:: +rewrite_index_flags(ostream &write) { + nassertv(_index_start != 0); -//////////////////////////////////////////////////////////////////// -// Function: Multifile::extract_all -// Access: Public -// Description: -//////////////////////////////////////////////////////////////////// -void Multifile:: -extract_all(const Filename &rel_path) { - express_cat.debug() - << "Multifile::extract_all() - Extracting all files" << endl; + static const size_t flags_offset = 4 + 4 + 4; + size_t flags_pos = _index_start + flags_offset; + write.seekp(flags_pos); + nassertv(!write.fail()); - MemfileList::iterator i; - for (i = _files.begin(); i != _files.end(); ++i) - (*i)->write(rel_path); + Datagram dg; + dg.add_uint16(_flags); + write.write(dg.get_data(), dg.get_length()); } diff --git a/panda/src/express/multifile.h b/panda/src/express/multifile.h index d6e7f87730..e2b3a7188d 100644 --- a/panda/src/express/multifile.h +++ b/panda/src/express/multifile.h @@ -15,21 +15,19 @@ // panda3d@yahoogroups.com . // //////////////////////////////////////////////////////////////////// + #ifndef MULTIFILE_H #define MULTIFILE_H -// -//////////////////////////////////////////////////////////////////// -// Includes -//////////////////////////////////////////////////////////////////// -#include -#include "typedef.h" +#include "pandabase.h" + #include "datagram.h" #include "datagramIterator.h" -#include -#include -#include "plist.h" +#include "filename.h" +#include "ordered_vector.h" +#include "indirectLess.h" +#include "pvector.h" //////////////////////////////////////////////////////////////////// // Class : Multifile @@ -37,95 +35,120 @@ //////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS Multifile { PUBLISHED: - enum Type { - T_unknown, - T_valid, - T_invalid, - }; + Multifile(); + ~Multifile(); - Multifile(void); - ~Multifile(void); + bool open_read(const Filename &multifile_name); + bool open_write(const Filename &multifile_name); + bool open_read_write(const Filename &multifile_name); + void close(); - INLINE int get_header_length(void) const; + INLINE bool is_read_valid() const; + INLINE bool is_write_valid() const; + INLINE bool needs_repack() const; - static int evaluate(const char *start, int size); + INLINE void set_scale_factor(size_t scale_factor); + INLINE size_t get_scale_factor() const; - bool add(const Filename &name); - bool remove(const Filename &name); - bool has_file(const Filename &name); + bool add_subfile(const string &subfile_name, const Filename &filename); + bool flush(); + bool repack(); - bool read(Filename &name); - bool write(Filename name); - int write(char *&start, int &size, const Filename &rel_path = ""); - bool write_extract(char *&start, int &size, const Filename &rel_path = ""); - bool extract(const Filename &name, const Filename &rel_path = ""); - void extract_all(const Filename &rel_path = ""); + int get_num_subfiles() const; + int find_subfile(const string &subfile_name) const; + void remove_subfile(int index); + const string &get_subfile_name(int index) const; + size_t get_subfile_length(int index) const; - void reset(void); - int parse_header(char *&start, int &size); - -private: - - INLINE void write_header(ofstream &write_stream); + void read_subfile(int index, Datagram &datagram); + bool extract_subfile(int index, const Filename &filename); public: - // This nested class must be public so we can make a list of its - // pointers. Weird. - class Memfile { - public: - Memfile(void); - ~Memfile(void); - bool parse_header_length(char *&start, int &size); - bool parse_header(char *&start, int &size); + // Special interfaces to work with iostreams, not necessarily files. + bool open_read(istream *multifile_stream); + bool open_write(ostream *multifile_stream); + bool open_read_write(iostream *multifile_stream); + bool add_subfile(const string &subfile_name, istream *subfile_data); - bool read(const Filename &name); - bool read_from_multifile(ifstream &read_stream); - bool write(const Filename &rel_path); - void write_to_multifile(ofstream &write_stream); - int write(char *&start, int &size, const Filename &rel_path = ""); - void reset(void); - - public: - bool _header_length_parsed; - bool _header_parsed; - Datagram _datagram; - char *_header_length_buf; - int _header_length_buf_length; - - PN_int32 _header_length; - Filename _name; - PN_int32 _buffer_length; - char* _buffer; - - bool _file_open; - ofstream _write_stream; - int _bytes_written; - }; + bool extract_subfile_to(int index, ostream &out); + istream &open_read_subfile(int index); + void close_subfile(); private: - typedef plist MemfileList; - - class MemfileMatch { - public: - MemfileMatch(const Filename &name) { - _want_name = name; - } - bool operator()(Memfile *mfile) const { - return mfile->_name == _want_name; - } - Filename _want_name; + enum SubfileFlags { + SF_deleted = 0x0001, + SF_index_invalid = 0x0002, + SF_data_invalid = 0x0004, }; -private: - MemfileList _files; - PN_int32 _num_mfiles; - bool _header_parsed; - Memfile *_current_mfile; - Datagram _datagram; - int _header_length; - int _mfiles_written; + class Subfile { + public: + INLINE Subfile(const string &name); + INLINE bool operator < (const Subfile &other) const; + streampos read_index(istream &read, streampos fpos, + Multifile *multfile); + streampos write_index(ostream &write, streampos fpos, + Multifile *multifile); + streampos write_data(ostream &write, istream *read, streampos fpos); + void rewrite_index_data_start(ostream &write, Multifile *multifile); + void rewrite_index_flags(ostream &write); + INLINE bool is_deleted() const; + INLINE bool is_index_invalid() const; + INLINE bool is_data_invalid() const; - static PN_uint32 _magic_number; + string _name; + streampos _index_start; + streampos _data_start; + size_t _data_length; + istream *_source; + Filename _source_filename; + int _flags; + }; + + INLINE streampos word_to_streampos(size_t word) const; + INLINE size_t streampos_to_word(streampos fpos) const; + INLINE streampos normalize_streampos(streampos fpos) const; + streampos pad_to_streampos(streampos fpos); + + bool add_new_subfile(Subfile *subfile); + void clear_subfiles(); + bool read_index(); + bool write_header(); + + + typedef ov_set > Subfiles; + Subfiles _subfiles; + typedef pvector PendingSubfiles; + PendingSubfiles _new_subfiles; + PendingSubfiles _removed_subfiles; + + istream *_read; + ostream *_write; + streampos _next_index; + streampos _last_index; + + bool _needs_repack; + size_t _scale_factor; + + ifstream _read_file; + ofstream _write_file; + fstream _read_write_file; + Filename _multifile_name; + + int _file_major_ver; + int _file_minor_ver; + + Subfile *_open_subfile; + // This is just to support open_read_subfile() for those subfiles + // that have recently been added but not flushed. + ifstream _subfile_read; + + static const char _header[]; + static const size_t _header_size; + static const int _current_major_ver; + static const int _current_minor_ver; + + friend class Subfile; }; #include "multifile.I" diff --git a/panda/src/putil/ordered_vector.I b/panda/src/express/ordered_vector.I similarity index 100% rename from panda/src/putil/ordered_vector.I rename to panda/src/express/ordered_vector.I diff --git a/panda/src/putil/ordered_vector.T b/panda/src/express/ordered_vector.T similarity index 100% rename from panda/src/putil/ordered_vector.T rename to panda/src/express/ordered_vector.T diff --git a/panda/src/putil/ordered_vector.cxx b/panda/src/express/ordered_vector.cxx similarity index 100% rename from panda/src/putil/ordered_vector.cxx rename to panda/src/express/ordered_vector.cxx diff --git a/panda/src/putil/ordered_vector.h b/panda/src/express/ordered_vector.h similarity index 99% rename from panda/src/putil/ordered_vector.h rename to panda/src/express/ordered_vector.h index 547a6fda09..80ed9e96f3 100644 --- a/panda/src/putil/ordered_vector.h +++ b/panda/src/express/ordered_vector.h @@ -20,7 +20,6 @@ #define ORDERED_VECTOR_H #include "pandabase.h" -#include "config_util.h" #include "pvector.h" #include "pset.h" diff --git a/panda/src/putil/test_ordered_vector.cxx b/panda/src/express/test_ordered_vector.cxx similarity index 100% rename from panda/src/putil/test_ordered_vector.cxx rename to panda/src/express/test_ordered_vector.cxx diff --git a/panda/src/putil/Sources.pp b/panda/src/putil/Sources.pp index 37741be6d6..1908d2e24b 100644 --- a/panda/src/putil/Sources.pp +++ b/panda/src/putil/Sources.pp @@ -30,13 +30,11 @@ globalPointerRegistry.I globalPointerRegistry.h \ indirectCompareNames.I indirectCompareNames.h \ indirectCompareTo.I indirectCompareTo.h \ - indirectLess.I indirectLess.h \ ioPtaDatagramFloat.h ioPtaDatagramInt.h \ ioPtaDatagramShort.h keyboardButton.h lineStream.I \ lineStream.h lineStreamBuf.I lineStreamBuf.h \ modifierButtons.I modifierButtons.h mouseButton.h \ mouseData.I mouseData.h nameUniquifier.I nameUniquifier.h \ - ordered_vector.h ordered_vector.I ordered_vector.T \ pipeline.h pipeline.I \ pipelineCycler.h pipelineCycler.I \ pipelineCyclerBase.h pipelineCyclerBase.I \ @@ -67,7 +65,6 @@ keyboardButton.cxx lineStream.cxx lineStreamBuf.cxx \ modifierButtons.cxx mouseButton.cxx mouseData.cxx \ nameUniquifier.cxx \ - ordered_vector.cxx \ pipeline.cxx \ pipelineCycler.cxx \ pipelineCyclerBase.cxx \ @@ -100,13 +97,12 @@ globPattern.I globPattern.h \ globalPointerRegistry.I globalPointerRegistry.h \ indirectCompareNames.I indirectCompareNames.h indirectCompareTo.I \ - indirectCompareTo.h indirectLess.I indirectLess.h \ + indirectCompareTo.h \ ioPtaDatagramFloat.h ioPtaDatagramInt.h \ ioPtaDatagramShort.h iterator_types.h keyboardButton.h lineStream.I \ lineStream.h lineStreamBuf.I lineStreamBuf.h modifierButtons.I \ modifierButtons.h mouseButton.h mouseData.I mouseData.h \ nameUniquifier.I nameUniquifier.h \ - ordered_vector.h ordered_vector.I ordered_vector.T \ pipeline.h pipeline.I \ pipelineCycler.h pipelineCycler.I \ pipelineCyclerBase.h pipelineCyclerBase.I \ @@ -170,14 +166,3 @@ test_linestream.cxx #end test_bin_target - -#begin test_bin_target - #define TARGET test_ordered_vector - - #define SOURCES \ - test_ordered_vector.cxx - - #define LOCAL_LIBS $[LOCAL_LIBS] putil - #define OTHER_LIBS $[OTHER_LIBS] pystub - -#end test_bin_target diff --git a/panda/src/putil/putil_composite2.cxx b/panda/src/putil/putil_composite2.cxx index d88413ed5d..881953fb73 100644 --- a/panda/src/putil/putil_composite2.cxx +++ b/panda/src/putil/putil_composite2.cxx @@ -1,5 +1,4 @@ #include "nameUniquifier.cxx" -#include "ordered_vector.cxx" #include "pipeline.cxx" #include "pipelineCycler.cxx" #include "pipelineCyclerBase.cxx"