From db1bc689e5fbe0fa470fcff3f9b6bec4deed7e3c Mon Sep 17 00:00:00 2001 From: David Rose Date: Tue, 4 Nov 2008 01:09:57 +0000 Subject: [PATCH] various fixes to threads, downloader, and support for VirtualFileMountHTTP --- panda/src/downloader/Sources.pp | 18 +- panda/src/downloader/bioStreamBuf.cxx | 33 +- panda/src/downloader/chunkedStream.cxx | 13 + panda/src/downloader/chunkedStream.h | 1 + panda/src/downloader/chunkedStreamBuf.cxx | 133 ++-- panda/src/downloader/chunkedStreamBuf.h | 1 + panda/src/downloader/config_downloader.cxx | 21 + panda/src/downloader/config_downloader.h | 2 + .../src/downloader/downloader_composite2.cxx | 4 + panda/src/downloader/httpChannel.I | 65 ++ panda/src/downloader/httpChannel.cxx | 578 ++++++++++++------ panda/src/downloader/httpChannel.h | 43 +- panda/src/downloader/httpClient.cxx | 14 + panda/src/downloader/httpClient.h | 22 +- panda/src/downloader/identityStream.I | 2 +- panda/src/downloader/identityStream.cxx | 13 + panda/src/downloader/identityStream.h | 1 + panda/src/downloader/identityStreamBuf.cxx | 24 +- panda/src/downloader/identityStreamBuf.h | 5 +- panda/src/downloader/socketStream.I | 1 + panda/src/downloader/socketStream.cxx | 28 + panda/src/downloader/socketStream.h | 15 + .../src/{putil => downloader}/stringStream.I | 0 .../{putil => downloader}/stringStream.cxx | 0 .../src/{putil => downloader}/stringStream.h | 2 +- .../{putil => downloader}/stringStreamBuf.I | 0 .../{putil => downloader}/stringStreamBuf.cxx | 0 .../{putil => downloader}/stringStreamBuf.h | 2 +- panda/src/downloader/virtualFileHTTP.I | 26 + panda/src/downloader/virtualFileHTTP.cxx | 278 +++++++++ panda/src/downloader/virtualFileHTTP.h | 89 +++ panda/src/downloader/virtualFileMountHTTP.I | 36 ++ panda/src/downloader/virtualFileMountHTTP.cxx | 227 +++++++ panda/src/downloader/virtualFileMountHTTP.h | 95 +++ panda/src/express/virtualFile.cxx | 27 +- panda/src/express/virtualFile.h | 2 + panda/src/express/virtualFileComposite.cxx | 10 + panda/src/express/virtualFileComposite.h | 1 + panda/src/express/virtualFileMount.I | 22 +- panda/src/express/virtualFileMount.cxx | 31 +- panda/src/express/virtualFileMount.h | 28 +- panda/src/express/virtualFileMountMultifile.I | 7 +- .../src/express/virtualFileMountMultifile.cxx | 10 + panda/src/express/virtualFileMountMultifile.h | 9 +- panda/src/express/virtualFileMountSystem.I | 18 +- panda/src/express/virtualFileMountSystem.cxx | 10 + panda/src/express/virtualFileMountSystem.h | 16 +- panda/src/express/virtualFileSimple.cxx | 10 + panda/src/express/virtualFileSimple.h | 1 + panda/src/express/virtualFileSystem.I | 3 +- panda/src/express/virtualFileSystem.cxx | 250 +++++--- panda/src/express/virtualFileSystem.h | 22 +- panda/src/gobj/texture.h | 2 +- panda/src/gobj/texturePool.I | 12 + panda/src/gobj/texturePool.cxx | 136 +++-- panda/src/gobj/texturePool.h | 14 +- panda/src/pipeline/conditionVarSimpleImpl.cxx | 6 +- panda/src/pipeline/mutexDebug.cxx | 4 +- panda/src/pipeline/mutexSimpleImpl.cxx | 2 +- panda/src/pipeline/threadSimpleImpl.I | 4 +- panda/src/pipeline/threadSimpleImpl.cxx | 8 +- panda/src/pipeline/threadSimpleImpl.h | 2 +- panda/src/pipeline/threadSimpleManager.cxx | 26 +- panda/src/pipeline/threadSimpleManager.h | 3 +- panda/src/putil/Sources.pp | 6 - panda/src/putil/putil_composite2.cxx | 2 - 66 files changed, 1965 insertions(+), 531 deletions(-) rename panda/src/{putil => downloader}/stringStream.I (100%) rename panda/src/{putil => downloader}/stringStream.cxx (100%) rename panda/src/{putil => downloader}/stringStream.h (95%) rename panda/src/{putil => downloader}/stringStreamBuf.I (100%) rename panda/src/{putil => downloader}/stringStreamBuf.cxx (100%) rename panda/src/{putil => downloader}/stringStreamBuf.h (96%) create mode 100644 panda/src/downloader/virtualFileHTTP.I create mode 100644 panda/src/downloader/virtualFileHTTP.cxx create mode 100644 panda/src/downloader/virtualFileHTTP.h create mode 100644 panda/src/downloader/virtualFileMountHTTP.I create mode 100644 panda/src/downloader/virtualFileMountHTTP.cxx create mode 100644 panda/src/downloader/virtualFileMountHTTP.h diff --git a/panda/src/downloader/Sources.pp b/panda/src/downloader/Sources.pp index 641f3e4f58..742a692c5b 100644 --- a/panda/src/downloader/Sources.pp +++ b/panda/src/downloader/Sources.pp @@ -36,7 +36,11 @@ patcher.h patcher.I \ socketStream.h socketStream.I \ ssl_utils.h \ - urlSpec.I urlSpec.h + stringStreamBuf.I stringStreamBuf.h \ + stringStream.I stringStream.h \ + urlSpec.I urlSpec.h \ + virtualFileHTTP.I virtualFileHTTP.h \ + virtualFileMountHTTP.I virtualFileMountHTTP.h #define INCLUDED_SOURCES \ config_downloader.cxx \ @@ -63,7 +67,11 @@ patcher.cxx \ socketStream.cxx \ ssl_utils.cxx \ - urlSpec.cxx + stringStreamBuf.cxx \ + stringStream.cxx \ + urlSpec.cxx \ + virtualFileHTTP.cxx \ + virtualFileMountHTTP.cxx #define INSTALL_HEADERS \ bioPtr.I bioPtr.h \ @@ -92,7 +100,11 @@ patcher.h patcher.I \ socketStream.h socketStream.I \ ssl_utils.h \ - urlSpec.h urlSpec.I + stringStreamBuf.I stringStreamBuf.h \ + stringStream.I stringStream.h \ + urlSpec.h urlSpec.I \ + virtualFileHTTP.I virtualFileHTTP.h \ + virtualFileMountHTTP.I virtualFileMountHTTP.h #define IGATESCAN all diff --git a/panda/src/downloader/bioStreamBuf.cxx b/panda/src/downloader/bioStreamBuf.cxx index 97eb39081b..a5c49df2bd 100644 --- a/panda/src/downloader/bioStreamBuf.cxx +++ b/panda/src/downloader/bioStreamBuf.cxx @@ -184,11 +184,11 @@ underflow() { int os_error = errno; #endif // WIN32_VC - // if not EOF, there may be more data on the way even if BIO - // indicates that no retry is necessary - //_is_closed = !BIO_should_retry(*_source); - //_is_closed = !BIO_should_read(*_source); - _is_closed = (BIO_eof(*_source) != 0); + // Though BIO_eof() is tempting, it appears there are cases in + // which that never returns true, if the socket is closed by + // the server. But BIO_should_retry() *appears* to be + // reliable. + _is_closed = !BIO_should_retry(*_source); if (_is_closed) { downloader_cat.info() << "Lost connection to " @@ -214,20 +214,19 @@ underflow() { } gbump(num_bytes); return EOF; - } - if (downloader_cat.is_spam()) { - downloader_cat.spam() - << "read " << read_count << " bytes from " << _source << "\n"; - } - // Slide what we did read to the top of the buffer. nassertr(read_count < (int)num_bytes, EOF); size_t delta = (int)num_bytes - read_count; memmove(gptr() + delta, gptr(), read_count); gbump(delta); } + + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "read " << read_count << " bytes from " << _source << "\n"; + } } return (unsigned char)*gptr(); @@ -246,10 +245,6 @@ write_chars(const char *start, size_t length) { size_t wrote_so_far = 0; int write_count = BIO_write(*_source, start, length); - if (downloader_cat.is_spam()) { - downloader_cat.spam() - << "wrote " << write_count << " bytes to " << _source << "\n"; - } thread_consider_yield(); while (write_count != (int)(length - wrote_so_far)) { if (write_count <= 0) { @@ -295,6 +290,10 @@ write_chars(const char *start, size_t length) { } else { // wrote some characters. wrote_so_far += write_count; + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "wrote " << write_count << " bytes to " << _source << "\n"; + } } // Try to write some more. @@ -305,6 +304,10 @@ write_chars(const char *start, size_t length) { } thread_consider_yield(); } + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "wrote " << write_count << " bytes to " << _source << "\n"; + } } return length; diff --git a/panda/src/downloader/chunkedStream.cxx b/panda/src/downloader/chunkedStream.cxx index 100c8328c4..698ae3b463 100644 --- a/panda/src/downloader/chunkedStream.cxx +++ b/panda/src/downloader/chunkedStream.cxx @@ -17,6 +17,19 @@ // This module is not compiled if OpenSSL is not available. #ifdef HAVE_OPENSSL +//////////////////////////////////////////////////////////////////// +// Function: IChunkedStream::Destructor +// Access: Published, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +IChunkedStream:: +~IChunkedStream() { + if (_channel != (HTTPChannel *)NULL) { + _channel->body_stream_destructs(this); + _channel = NULL; + } +} + //////////////////////////////////////////////////////////////////// // Function: IChunkedStream::is_closed // Access: Public, Virtual diff --git a/panda/src/downloader/chunkedStream.h b/panda/src/downloader/chunkedStream.h index 67d775986c..a0a8a22e76 100644 --- a/panda/src/downloader/chunkedStream.h +++ b/panda/src/downloader/chunkedStream.h @@ -41,6 +41,7 @@ public: INLINE IChunkedStream(BioStreamPtr *source, HTTPChannel *doc); INLINE IChunkedStream &open(BioStreamPtr *source, HTTPChannel *doc); + virtual ~IChunkedStream(); virtual bool is_closed(); virtual void close(); diff --git a/panda/src/downloader/chunkedStreamBuf.cxx b/panda/src/downloader/chunkedStreamBuf.cxx index a711d2f946..8a9ea4079e 100644 --- a/panda/src/downloader/chunkedStreamBuf.cxx +++ b/panda/src/downloader/chunkedStreamBuf.cxx @@ -33,6 +33,7 @@ ChunkedStreamBuf:: ChunkedStreamBuf() { _chunk_remaining = 0; _done = true; + _wanted_nonblocking = false; _read_state = ISocketStream::RS_initial; #ifdef HAVE_IOSTREAM @@ -74,6 +75,7 @@ open_read(BioStreamPtr *source, HTTPChannel *doc) { nassertv(!_source.is_null()); _chunk_remaining = 0; _done = false; + _wanted_nonblocking = doc->_wanted_nonblocking; _read_state = ISocketStream::RS_reading; _doc = doc; @@ -141,64 +143,87 @@ underflow() { //////////////////////////////////////////////////////////////////// size_t ChunkedStreamBuf:: read_chars(char *start, size_t length) { - nassertr(!_source.is_null(), 0); - if (_done) { - return 0; - } - - if (_chunk_remaining != 0) { - // Extract some of the bytes remaining in the chunk. - length = min(length, _chunk_remaining); - (*_source)->read(start, length); - size_t read_count = (*_source)->gcount(); - _chunk_remaining -= read_count; - - if (read_count == 0 && (*_source)->is_closed()) { - // Whoops, the socket closed while we were downloading. - _read_state = ISocketStream::RS_error; + while (true) { + nassertr(!_source.is_null(), 0); + if (_done) { + return 0; } - - return read_count; - } - - // Read the next chunk. - string line; - bool got_line = http_getline(line); - while (got_line && line.empty()) { - // Skip blank lines. There really should be exactly one blank - // line, but who's counting? It's tricky to count and maintain - // reentry for nonblocking I/O. - got_line = http_getline(line); - } - if (!got_line) { - // EOF (or data unavailable) while trying to read the chunk size. - if ((*_source)->is_closed()) { - // Whoops, the socket closed while we were downloading. - _read_state = ISocketStream::RS_error; + + if (_chunk_remaining != 0) { + // Extract some of the bytes remaining in the chunk. + length = min(length, _chunk_remaining); + (*_source)->read(start, length); + size_t read_count = (*_source)->gcount(); + if (!_wanted_nonblocking) { + while (read_count == 0 && !(*_source)->is_closed()) { + // Simulate blocking. + thread_yield(); + (*_source)->read(start, length); + read_count = (*_source)->gcount(); + } + } + _chunk_remaining -= read_count; + + if (read_count == 0 && (*_source)->is_closed()) { + // Whoops, the socket closed while we were downloading. + _read_state = ISocketStream::RS_error; + } + + return read_count; } - return 0; - } - size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16); - if (downloader_cat.is_spam()) { - downloader_cat.spam() - << "Got chunk of size " << chunk_size << " bytes.\n"; + + // Read the next chunk. + string line; + bool got_line = http_getline(line); + while (got_line && line.empty()) { + // Skip blank lines. There really should be exactly one blank + // line, but who's counting? It's tricky to count and maintain + // reentry for nonblocking I/O. + got_line = http_getline(line); + } + if (!got_line) { + // EOF (or data unavailable) while trying to read the chunk size. + if ((*_source)->is_closed()) { + // Whoops, the socket closed while we were downloading. + _read_state = ISocketStream::RS_error; + } + + if (!_wanted_nonblocking) { + // Simulate blocking. + thread_yield(); + continue; // back to the top. + } + + return 0; + } + size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16); + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "Got chunk of size " << chunk_size << " bytes.\n"; + } + + if (chunk_size == 0) { + // Last chunk; we're done. + _done = true; + if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { + _doc->_file_size = _doc->_transfer_file_size; + _doc->_got_file_size = true; + } + _read_state = ISocketStream::RS_complete; + return 0; + } + + if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { + _doc->_transfer_file_size += chunk_size; + } + + _chunk_remaining = chunk_size; + + // Back to the top. } - if (chunk_size == 0) { - // Last chunk; we're done. - _done = true; - _doc->_file_size = _doc->_transfer_file_size; - _doc->_got_file_size = true; - _read_state = ISocketStream::RS_complete; - return 0; - } - - if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { - _doc->_transfer_file_size += chunk_size; - } - - _chunk_remaining = chunk_size; - return read_chars(start, length); + // Never gets here. + return 0; } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/downloader/chunkedStreamBuf.h b/panda/src/downloader/chunkedStreamBuf.h index bbd590d728..630bebc66d 100644 --- a/panda/src/downloader/chunkedStreamBuf.h +++ b/panda/src/downloader/chunkedStreamBuf.h @@ -51,6 +51,7 @@ private: PT(BioStreamPtr) _source; size_t _chunk_remaining; bool _done; + bool _wanted_nonblocking; string _working_getline; ISocketStream::ReadState _read_state; diff --git a/panda/src/downloader/config_downloader.cxx b/panda/src/downloader/config_downloader.cxx index 6cc79f4955..ed1bb58fb0 100644 --- a/panda/src/downloader/config_downloader.cxx +++ b/panda/src/downloader/config_downloader.cxx @@ -15,6 +15,8 @@ #include "dconfig.h" #include "config_downloader.h" #include "httpChannel.h" +#include "virtualFileHTTP.h" +#include "virtualFileMountHTTP.h" #include "pandaSystem.h" @@ -114,6 +116,23 @@ ConfigVariableDouble http_timeout "It starts counting after the TCP connection has been established " "(http_connect_timeout, above) and the request has been sent.")); +ConfigVariableInt http_skip_body_size +("http-skip-body-size", 8192, + PRC_DESC("This is the maximum number of bytes in a received " + "(but unwanted) body that will be skipped past, in " + "order to reset to a new request. " + "See HTTPChannel::set_skip_body_size().")); + +ConfigVariableDouble http_idle_timeout +("http-idle-timeout", 5.0, + PRC_DESC("This the amount of time, in seconds, in which a " + "previously-established connection is allowed to remain open " + "and unused. If a previous connection has remained unused for " + "at least this number of seconds, it will be closed and a new " + "connection will be opened; otherwise, the same connection " + "will be reused for the next request (for a particular " + "HTTPChannel).")); + ConfigVariableInt http_max_connect_count ("http-max-connect-count", 10, PRC_DESC("This is the maximum number of times to try reconnecting to the " @@ -166,6 +185,8 @@ init_libdownloader() { #ifdef HAVE_OPENSSL HTTPChannel::init_type(); + VirtualFileHTTP::init_type(); + VirtualFileMountHTTP::init_type(); // We need to define this here, rather than above, to guarantee that // it has been initialized by the time we check it. diff --git a/panda/src/downloader/config_downloader.h b/panda/src/downloader/config_downloader.h index ee14a54cc3..24ad92ac13 100644 --- a/panda/src/downloader/config_downloader.h +++ b/panda/src/downloader/config_downloader.h @@ -50,6 +50,8 @@ extern ConfigVariableString http_proxy_username; extern ConfigVariableBool http_proxy_tunnel; extern ConfigVariableDouble http_connect_timeout; extern ConfigVariableDouble http_timeout; +extern ConfigVariableInt http_skip_body_size; +extern ConfigVariableDouble http_idle_timeout; extern ConfigVariableInt http_max_connect_count; extern ConfigVariableFilename http_client_certificate_filename; extern ConfigVariableString http_client_certificate_passphrase; diff --git a/panda/src/downloader/downloader_composite2.cxx b/panda/src/downloader/downloader_composite2.cxx index f02cb56782..b7f03d1b35 100644 --- a/panda/src/downloader/downloader_composite2.cxx +++ b/panda/src/downloader/downloader_composite2.cxx @@ -14,4 +14,8 @@ #include "patcher.cxx" #include "socketStream.cxx" #include "ssl_utils.cxx" +#include "stringStreamBuf.cxx" +#include "stringStream.cxx" #include "urlSpec.cxx" +#include "virtualFileHTTP.cxx" +#include "virtualFileMountHTTP.cxx" diff --git a/panda/src/downloader/httpChannel.I b/panda/src/downloader/httpChannel.I index 10591843b6..b64d21cb76 100644 --- a/panda/src/downloader/httpChannel.I +++ b/panda/src/downloader/httpChannel.I @@ -391,6 +391,71 @@ get_http_timeout() const { return _http_timeout; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::set_skip_body_size +// Access: Published +// Description: Specifies the maximum number of bytes in a received +// (but unwanted) body that will be skipped past, in +// order to reset to a new request. +// +// That is, if this HTTPChannel requests a file via +// get_document(), but does not call download_to_ram(), +// download_to_file(), or open_read_body(), and instead +// immediately requests a new file, then the HTTPChannel +// has a choice whether to skip past the unwanted +// document, or to close the connection and open a new +// one. If the number of bytes to skip is more than +// this threshold, the connection will be closed; +// otherwise, the data will simply be read and +// discarded. +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +set_skip_body_size(size_t skip_body_size) { + _skip_body_size = skip_body_size; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::get_skip_body_size +// Access: Published +// Description: Returns the maximum number of bytes in a received +// (but unwanted) body that will be skipped past, in +// order to reset to a new request. See +// set_skip_body_size(). +//////////////////////////////////////////////////////////////////// +INLINE size_t HTTPChannel:: +get_skip_body_size() const { + return _skip_body_size; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::set_idle_timeout +// Access: Published +// Description: Specifies the amount of time, in seconds, in which a +// previously-established connection is allowed to +// remain open and unused. If a previous connection has +// remained unused for at least this number of seconds, +// it will be closed and a new connection will be +// opened; otherwise, the same connection will be reused +// for the next request (for this particular +// HTTPChannel). +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +set_idle_timeout(double idle_timeout) { + _idle_timeout = idle_timeout; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::get_idle_timeout +// Access: Published +// Description: Returns the amount of time, in seconds, in which an +// previously-established connection is allowed to +// remain open and unused. See set_idle_timeout(). +//////////////////////////////////////////////////////////////////// +INLINE double HTTPChannel:: +get_idle_timeout() const { + return _idle_timeout; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::set_download_throttle // Access: Published diff --git a/panda/src/downloader/httpChannel.cxx b/panda/src/downloader/httpChannel.cxx index 4c983f05c2..e602a3aa83 100644 --- a/panda/src/downloader/httpChannel.cxx +++ b/panda/src/downloader/httpChannel.cxx @@ -20,6 +20,7 @@ #include "chunkedStream.h" #include "identityStream.h" #include "config_downloader.h" +#include "virtualFileMountHTTP.h" #include "ramfile.h" #ifdef HAVE_OPENSSL @@ -48,6 +49,8 @@ HTTPChannel(HTTPClient *client) : _proxy_tunnel = http_proxy_tunnel; _connect_timeout = http_connect_timeout; _http_timeout = http_timeout; + _skip_body_size = http_skip_body_size; + _idle_timeout = http_idle_timeout; _blocking_connect = false; _download_throttle = false; _max_bytes_per_second = downloader_byte_rate; @@ -89,6 +92,7 @@ HTTPChannel(HTTPClient *client) : _started_download = false; _sent_so_far = 0; _body_stream = NULL; + _owns_body_stream = false; _sbio = NULL; _last_status_code = 0; _last_run_time = 0.0f; @@ -106,108 +110,6 @@ HTTPChannel:: reset_download_to(); } -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::get_file_system -// Access: Published, Virtual -// Description: Returns the VirtualFileSystem this file is associated -// with. -//////////////////////////////////////////////////////////////////// -VirtualFileSystem *HTTPChannel:: -get_file_system() const { - return NULL; -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::get_filename -// Access: Published, Virtual -// Description: Returns the full pathname to this file within the -// virtual file system. -//////////////////////////////////////////////////////////////////// -Filename HTTPChannel:: -get_filename() const { - return Filename(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::is_regular_file -// Access: Published, Virtual -// Description: Returns true if this file represents a regular file -// (and read_file() may be called), false otherwise. -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -is_regular_file() const { - return is_valid(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::will_close_connection -// Access: Public -// Description: Returns true if the server has indicated it will -// close the connection after this document has been -// read, or false if it will remain open (and future -// documents may be requested on the same connection). -//////////////////////////////////////////////////////////////////// -bool HTTPChannel:: -will_close_connection() const { - if (get_http_version() < HTTPEnum::HV_11) { - // pre-HTTP 1.1 always closes. - return true; - } - - string connection = get_header_value("Connection"); - if (downcase(connection) == "close") { - // The server says it will close. - return true; - } - - if (connection.empty() && !get_persistent_connection()) { - // The server didn't say, but we asked it to close. - return true; - } - - // Assume the server will keep it open. - return false; -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::open_read_file -// Access: Public, Virtual -// Description: Opens the document for reading. Returns a newly -// allocated istream on success (which you should -// pass to close_read_file() when you are done reading). -// Returns NULL on failure. This may only be called -// once for a particular HTTPChannel. -//////////////////////////////////////////////////////////////////// -istream *HTTPChannel:: -open_read_file(bool auto_unwrap) const { - return ((HTTPChannel *)this)->read_body(); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::close_read_file -// Access: Public -// Description: Closes a file opened by a previous call to -// open_read_file(). This really just deletes the -// istream pointer, but it is recommended to use this -// interface instead of deleting it explicitly, to help -// work around compiler issues. -//////////////////////////////////////////////////////////////////// -void HTTPChannel:: -close_read_file(istream *stream) const { - if (stream != (istream *)NULL) { - // For some reason--compiler bug in gcc 3.2?--explicitly deleting - // the stream pointer does not call the appropriate global delete - // function; instead apparently calling the system delete - // function. So we call the delete function by hand instead. -#if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW) - stream->~istream(); - (*global_operator_delete)(stream); -#else - delete stream; -#endif - } -} - //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::get_status_string // Access: Published @@ -296,6 +198,36 @@ get_header_value(const string &key) const { return string(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::will_close_connection +// Access: Published +// Description: Returns true if the server has indicated it will +// close the connection after this document has been +// read, or false if it will remain open (and future +// documents may be requested on the same connection). +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +will_close_connection() const { + if (get_http_version() < HTTPEnum::HV_11) { + // pre-HTTP 1.1 always closes. + return true; + } + + string connection = get_header_value("Connection"); + if (downcase(connection) == "close") { + // The server says it will close. + return true; + } + + if (connection.empty() && !get_persistent_connection()) { + // The server didn't say, but we asked it to close. + return true; + } + + // Assume the server will keep it open. + return false; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::get_file_size // Access: Published, Virtual @@ -306,7 +238,7 @@ get_header_value(const string &key) const { // // If the file is dynamically generated, the size may // not be available until a read has started -// (e.g. open_read_file() has been called); and even +// (e.g. open_read_body() has been called); and even // then it may increase as more of the file is read due // to the nature of HTTP/1.1 requests which can change // their minds midstream about how much data they're @@ -394,6 +326,10 @@ run() { case DD_ram: repeat_later = run_download_to_ram(); break; + + case DD_stream: + repeat_later = run_download_to_stream(); + break; } if (repeat_later) { thread_yield(); @@ -573,7 +509,7 @@ run() { } //////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::read_body +// Function: HTTPChannel::open_read_body // Access: Published // Description: Returns a newly-allocated istream suitable for // reading the body of the document. This may only be @@ -581,11 +517,19 @@ run() { // post_form(), or after a call to run() has returned // false. // -// The user is responsible for deleting the returned -// istream later. +// Note that, in nonblocking mode, the returned stream +// may report an early EOF, even before the actual end +// of file. When this happens, you should call +// stream->is_closed() to determine whether you should +// attempt to read some more later. +// +// The user is responsible for passing the returned +// istream to close_read_body() later. //////////////////////////////////////////////////////////////////// ISocketStream *HTTPChannel:: -read_body() { +open_read_body() { + reset_body_stream(); + if ((_state != S_read_header && _state != S_begin_body) || _source.is_null()) { return NULL; } @@ -611,9 +555,38 @@ read_body() { result = new IIdentityStream(_source, this, _got_file_size, _file_size); } + result->_channel = this; + _body_stream = result; + _owns_body_stream = false; + return result; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::close_read_body +// Access: Public +// Description: Closes a file opened by a previous call to +// open_read_body(). This really just deletes the +// istream pointer, but it is recommended to use this +// interface instead of deleting it explicitly, to help +// work around compiler issues. +//////////////////////////////////////////////////////////////////// +void HTTPChannel:: +close_read_body(istream *stream) const { + if (stream != (istream *)NULL) { + // For some reason--compiler bug in gcc 3.2?--explicitly deleting + // the stream pointer does not call the appropriate global delete + // function; instead apparently calling the system delete + // function. So we call the delete function by hand instead. +#if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW) + stream->~istream(); + (*global_operator_delete)(stream); +#else + delete stream; +#endif + } +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::download_to_file // Access: Published @@ -733,6 +706,66 @@ download_to_ram(Ramfile *ramfile, bool subdocument_resumes) { return is_download_complete(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::download_to_stream +// Access: Published +// Description: Specifies the name of an ostream to download the +// resulting document to. This should be called +// immediately after get_document() or +// begin_get_document() or related functions. +// +// In the case of the blocking I/O methods like +// get_document(), this function will download the +// entire document to the file and return true if it was +// successfully downloaded, false otherwise. +// +// In the case of non-blocking I/O methods like +// begin_get_document(), this function simply indicates an +// intention to download to the indicated file. It +// returns true if the file can be opened for writing, +// false otherwise, but the contents will not be +// completely downloaded until run() has returned false. +// At this time, it is possible that a communications +// error will have left a partial file, so +// is_download_complete() may be called to test this. +// +// If subdocument_resumes is true and the document in +// question was previously requested as a subdocument +// (i.e. get_subdocument() with a first_byte value +// greater than zero), this will automatically seek to +// the appropriate byte within the file for writing the +// output. In this case, the file must already exist +// and must have at least first_byte bytes in it. If +// subdocument_resumes is false, a subdocument will +// always be downloaded beginning at the first byte of +// the file. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +download_to_stream(ostream *strm, bool subdocument_resumes) { + reset_download_to(); + _download_to_stream = strm; + _download_to_stream->clear(); + _subdocument_resumes = subdocument_resumes; + + _download_dest = DD_stream; + + if (_wanted_nonblocking) { + // In nonblocking mode, we can't start the download yet; that will + // be done later as run() is called. + return true; + } + + // In normal, blocking mode, go ahead and do the download. + if (!open_download_file()) { + reset_download_to(); + return false; + } + + while (run()) { + } + return is_download_complete(); +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::get_connection // Access: Published @@ -754,7 +787,11 @@ get_connection() { BioStream *stream = _source->get_stream(); _source->set_stream(NULL); - // We're now passing ownership of the connection to the user. + // We're now passing ownership of the connection to the caller. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "passing ownership of connection to caller.\n"; + } reset_to_new(); return stream; @@ -777,6 +814,35 @@ downcase(const string &s) { return result; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::body_stream_destructs +// Access: Public +// Description: Called by ISocketStream destructor when _body_stream +// is destructing. +//////////////////////////////////////////////////////////////////// +void HTTPChannel:: +body_stream_destructs(ISocketStream *stream) { + if (stream == _body_stream) { + if (_state == S_reading_body) { + switch (_body_stream->get_read_state()) { + case ISocketStream::RS_complete: + finished_body(false); + break; + + case ISocketStream::RS_error: + _state = HTTPChannel::S_failure; + _status_entry._status_code = HTTPChannel::SC_lost_connection; + break; + + default: + break; + } + } + _body_stream = NULL; + _owns_body_stream = false; + } +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::reached_done_state @@ -842,13 +908,7 @@ reached_done_state() { } else { // Oops, we have to download the body now. - // We shouldn't already be in the middle of reading some other - // body when we come here. - if (_body_stream != NULL) { - delete _body_stream; - _body_stream = (ISocketStream *)NULL; - } - _body_stream = read_body(); + open_read_body(); if (_body_stream == (ISocketStream *)NULL) { if (downloader_cat.is_debug()) { downloader_cat.debug() @@ -857,8 +917,9 @@ reached_done_state() { return false; } else { + _owns_body_stream = true; if (_state != S_reading_body) { - _body_stream = NULL; + reset_body_stream(); } _started_download = true; @@ -940,7 +1001,11 @@ run_connecting() { } } else { - _state = _want_ssl ? S_setup_ssl : S_ready; + if (_want_ssl) { + _state = S_setup_ssl; + } else { + _state = S_ready; + } } return false; } @@ -1124,7 +1189,12 @@ run_http_proxy_reading_header() { // Now we have a tunnel opened through the proxy. make_request_text(); - _state = _want_ssl ? S_setup_ssl : S_ready; + if (_want_ssl) { + _state = S_setup_ssl; + } else { + _state = S_ready; + } + return false; } @@ -1370,7 +1440,12 @@ run_socks_proxy_connect_reply() { << connect_port << "\n"; } - _state = _want_ssl ? S_setup_ssl : S_ready; + if (_want_ssl) { + _state = S_setup_ssl; + } else { + _state = S_ready; + } + return false; } @@ -1927,6 +2002,10 @@ run_begin_body() { if (will_close_connection()) { // If the socket will close anyway, no point in skipping past the // previous body; just reset. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to begin body; server would close anyway.\n"; + } reset_to_new(); return false; } @@ -1935,11 +2014,10 @@ run_begin_body() { // We have already "read" the nonexistent body. _state = S_read_trailer; - } else if (get_file_size() > 8192) { + } else if (get_file_size() > _skip_body_size) { // If we know the size of the body we are about to skip and it's - // too large (and here we arbitrarily say 8KB is too large), then - // don't bother skipping it--just drop the connection and get a - // new one. + // too large, then don't bother skipping it--just drop the + // connection and get a new one. if (downloader_cat.is_debug()) { downloader_cat.debug() << "Dropping connection rather than skipping past " @@ -1948,13 +2026,7 @@ run_begin_body() { reset_to_new(); } else { - // We shouldn't already be in the middle of reading some other - // body when we come here. - if (_body_stream != NULL) { - delete _body_stream; - _body_stream = (ISocketStream *)NULL; - } - _body_stream = read_body(); + open_read_body(); if (_body_stream == (ISocketStream *)NULL) { if (downloader_cat.is_debug()) { downloader_cat.debug() @@ -1963,8 +2035,9 @@ run_begin_body() { reset_to_new(); } else { + _owns_body_stream = true; if (_state != S_reading_body) { - _body_stream = NULL; + reset_body_stream(); } } } @@ -1978,8 +2051,7 @@ run_begin_body() { // Description: In this state we are in the process of reading the // response's body. We will only come to this function // if the user did not choose to read the entire body -// himself (by calling read_body(), for instance, or -// open_read_file()). +// himself (by calling open_read_body()). // // In this case we should skip past the body to reset // the connection for making a new request. @@ -1989,13 +2061,21 @@ run_reading_body() { if (will_close_connection()) { // If the socket will close anyway, no point in skipping past the // previous body; just reset. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to read body; server would close anyway.\n"; + } reset_to_new(); return false; } // Skip the body we've already started. - if (_body_stream == NULL) { + if (_body_stream == NULL || !_owns_body_stream) { // Whoops, we're not in skip-body mode. Better reset. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting, not in skip-body mode.\n"; + } reset_to_new(); return false; } @@ -2009,29 +2089,14 @@ run_reading_body() { getline(*_body_stream, line); } - switch (_body_stream->get_read_state()) { - case ISocketStream::RS_complete: - finished_body(false); - break; - - case ISocketStream::RS_error: - _state = HTTPChannel::S_failure; - _status_entry._status_code = HTTPChannel::SC_lost_connection; - break; - - default: - break; - } - if (!_body_stream->is_closed()) { // There's more to come later. return true; } - delete _body_stream; - _body_stream = NULL; + reset_body_stream(); - // This should have been set by the _body_stream finishing. + // This should have been set by the call to finished_body(), above. nassertr(_state != S_reading_body, false); return false; } @@ -2055,6 +2120,10 @@ run_read_body() { if (will_close_connection()) { // If the socket will close anyway, no point in skipping past the // previous body; just reset. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to read body; server would close anyway.\n"; + } reset_to_new(); return false; } @@ -2086,6 +2155,10 @@ run_read_trailer() { if (will_close_connection()) { // If the socket will close anyway, no point in skipping past the // previous body; just reset. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to read trailer; server would close anyway.\n"; + } reset_to_new(); return false; } @@ -2102,7 +2175,7 @@ run_read_trailer() { //////////////////////////////////////////////////////////////////// bool HTTPChannel:: run_download_to_file() { - nassertr(_body_stream != (ISocketStream *)NULL, false); + nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false); bool do_throttle = _wanted_nonblocking && _download_throttle; @@ -2143,22 +2216,9 @@ run_download_to_file() { _download_to_file.flush(); - switch (_body_stream->get_read_state()) { - case ISocketStream::RS_complete: - finished_body(false); - break; - - case ISocketStream::RS_error: - _state = HTTPChannel::S_failure; - _status_entry._status_code = HTTPChannel::SC_lost_connection; - break; - - default: - break; - } - if (_body_stream->is_closed()) { // Done. + reset_body_stream(); _download_to_file.close(); _started_download = false; return false; @@ -2176,7 +2236,7 @@ run_download_to_file() { //////////////////////////////////////////////////////////////////// bool HTTPChannel:: run_download_to_ram() { - nassertr(_body_stream != (ISocketStream *)NULL, false); + nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false); nassertr(_download_to_ramfile != (Ramfile *)NULL, false); bool do_throttle = _wanted_nonblocking && _download_throttle; @@ -2207,22 +2267,70 @@ run_download_to_ram() { count = _body_stream->gcount(); } - switch (_body_stream->get_read_state()) { - case ISocketStream::RS_complete: - finished_body(false); - break; - - case ISocketStream::RS_error: - _state = HTTPChannel::S_failure; - _status_entry._status_code = HTTPChannel::SC_lost_connection; - break; - - default: - break; + if (_body_stream->is_closed()) { + // Done. + reset_body_stream(); + _started_download = false; + return false; + } else { + // More to come. + return true; } +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::run_download_to_stream +// Access: Private +// Description: After the headers, etc. have been read, this streams +// the download to the named file. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +run_download_to_stream() { + nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false); + + bool do_throttle = _wanted_nonblocking && _download_throttle; + + static const size_t buffer_size = 1024; + char buffer[buffer_size]; + + size_t remaining_this_pass = buffer_size; + if (do_throttle) { + remaining_this_pass = _bytes_per_update; + } + + _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); + size_t count = _body_stream->gcount(); + while (count != 0) { + _download_to_stream->write(buffer, count); + _bytes_downloaded += count; + if (do_throttle) { + nassertr(count <= remaining_this_pass, false); + remaining_this_pass -= count; + if (remaining_this_pass == 0) { + // That's enough for now. + return true; + } + } + + _body_stream->read(buffer, min(buffer_size, remaining_this_pass)); + count = _body_stream->gcount(); + } + + if (_download_to_stream->fail()) { + downloader_cat.warning() + << "Error writing to stream\n"; + _status_entry._status_code = SC_download_write_error; + _state = S_failure; + reset_download_to(); + return false; + } + + _download_to_stream->flush(); if (_body_stream->is_closed()) { // Done. + reset_body_stream(); + _download_to_stream = NULL; _started_download = false; return false; } else { @@ -2281,12 +2389,20 @@ begin_request(HTTPEnum::Method method, const DocumentSpec &url, if (_proxy != new_proxy) { _proxy = new_proxy; _proxy_auth = (HTTPAuthorization *)NULL; + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to change proxy to " << _proxy << "\n"; + } reset_to_new(); } // Ditto with changing the nonblocking state. if (_nonblocking != nonblocking) { _nonblocking = nonblocking; + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to change nonblocking state to " << _nonblocking << ".\n"; + } reset_to_new(); } @@ -2308,6 +2424,19 @@ begin_request(HTTPEnum::Method method, const DocumentSpec &url, // Also, reset from whatever previous request might still be pending. if (_state == S_failure || (_state < S_read_header && _state != S_ready)) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to clear previous request.\n"; + } + reset_to_new(); + + } else if (TrueClock::get_global_ptr()->get_short_time() - _last_run_time >= _idle_timeout) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting old connection: " + << TrueClock::get_global_ptr()->get_short_time() - _last_run_time + << " s old.\n"; + } reset_to_new(); } else if (_state == S_read_header) { @@ -2316,7 +2445,11 @@ begin_request(HTTPEnum::Method method, const DocumentSpec &url, _state = S_begin_body; } - _done_state = (_method == HTTPEnum::M_connect) ? S_ready : S_read_header; + if (_method == HTTPEnum::M_connect) { + _done_state = S_ready; + } else { + _done_state = S_read_header; + } } //////////////////////////////////////////////////////////////////// @@ -2381,6 +2514,7 @@ reconsider_proxy() { void HTTPChannel:: reset_for_new_request() { reset_download_to(); + reset_body_stream(); _last_status_code = 0; _status_entry = StatusEntry(); @@ -2406,6 +2540,10 @@ reset_for_new_request() { void HTTPChannel:: finished_body(bool has_trailer) { if (will_close_connection() && _download_dest == DD_none) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting to finish body; server would close anyway.\n"; + } reset_to_new(); } else { @@ -2421,12 +2559,12 @@ finished_body(bool has_trailer) { // Function: HTTPChannel::open_download_file // Access: Private // Description: If a download has been requested, opens the file on -// disk (or prepares the RamFile) and seeks within it to -// the appropriate _first_byte_delivered position, so -// that downloaded bytes will be written to the -// appropriate point within the file. Returns true if -// the starting position is valid, false otherwise (in -// which case the state is set to S_failure). +// disk (or prepares the RamFile or stream) and seeks +// within it to the appropriate _first_byte_delivered +// position, so that downloaded bytes will be written to +// the appropriate point within the file. Returns true +// if the starting position is valid, false otherwise +// (in which case the state is set to S_failure). //////////////////////////////////////////////////////////////////// bool HTTPChannel:: open_download_file() { @@ -2479,6 +2617,24 @@ open_download_file() { _download_to_ramfile->_data = _download_to_ramfile->_data.substr(0, _first_byte_delivered); } + } else if (_download_dest == DD_stream) { + // Windows doesn't complain if you try to seek past the end of + // file--it happily appends enough zero bytes to make the + // difference. Blecch. That means we need to get the file size + // first to check it ourselves. + _download_to_stream->seekp(0, ios::end); + if (_first_byte_delivered > (size_t)_download_to_stream->tellp()) { + downloader_cat.info() + << "Invalid starting position of byte " << _first_byte_delivered + << " within stream (which has " + << _download_to_stream->tellp() << " bytes)\n"; + _download_to_stream = NULL; + _status_entry._status_code = SC_download_invalid_range; + _state = S_failure; + return false; + } + + _download_to_stream->seekp(_first_byte_delivered); } } else { @@ -2489,6 +2645,8 @@ open_download_file() { _download_to_file.seekp(0); } else if (_download_dest == DD_ram) { _download_to_ramfile->_data = string(); + } else if (_download_dest == DD_stream) { + _download_to_stream->seekp(0); } } @@ -2523,8 +2681,8 @@ server_getline(string &str) { } str = str.substr(0, p); } - if (downloader_cat.is_spam()) { - downloader_cat.spam() << "recv: " << str << "\n"; + if (downloader_cat.is_debug()) { + downloader_cat.debug() << "recv: " << str << "\n"; } return true; @@ -2697,9 +2855,14 @@ server_send(const string &str, bool secret) { reset_to_new(); return false; } + + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "wrote " << write_count << " bytes to " << _bio << "\n"; + } #ifndef NDEBUG - if (!secret && downloader_cat.is_spam()) { + if (!secret && downloader_cat.is_debug()) { show_send(str.substr(0, write_count)); } #endif @@ -2736,6 +2899,10 @@ parse_http_response(const string &line) { } else { // Maybe we were just in some bad state. Drop the connection // and try again, once. + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "got non-HTTP response, resetting.\n"; + } reset_to_new(); _response_type = RT_non_http; } @@ -3416,6 +3583,11 @@ reset_url(const URLSpec &old_url, const URLSpec &new_url) { if (new_url.get_scheme() != old_url.get_scheme() || (_proxy.empty() && (new_url.get_server() != old_url.get_server() || new_url.get_port() != old_url.get_port()))) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "resetting for new server " + << new_url.get_server_and_port() << "\n"; + } reset_to_new(); } } @@ -3458,14 +3630,14 @@ show_send(const string &message) { size_t newline = message.find('\n', start); while (newline != string::npos) { // Assume every \n is preceded by a \r. - downloader_cat.spam() + downloader_cat.debug() << "send: " << message.substr(start, newline - start - 1) << "\n"; start = newline + 1; newline = message.find('\n', start); } if (start < message.length()) { - downloader_cat.spam() + downloader_cat.debug() << "send: " << message.substr(start) << " (no newline)\n"; } } @@ -3483,6 +3655,7 @@ reset_download_to() { _started_download = false; _download_to_file.close(); _download_to_ramfile = (Ramfile *)NULL; + _download_to_stream = NULL; _download_dest = DD_none; } @@ -3497,6 +3670,24 @@ reset_to_new() { _state = S_new; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::reset_body_stream +// Access: Private +// Description: Clears the _body_stream pointer, if it is set. +//////////////////////////////////////////////////////////////////// +void HTTPChannel:: +reset_body_stream() { + if (_owns_body_stream) { + if (_body_stream != (ISocketStream *)NULL) { + close_read_body(_body_stream); + nassertv(_body_stream == (ISocketStream *)NULL && !_owns_body_stream); + } + } else { + _body_stream = NULL; + } +} + + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::close_connection // Access: Private @@ -3505,10 +3696,7 @@ reset_to_new() { //////////////////////////////////////////////////////////////////// void HTTPChannel:: close_connection() { - if (_body_stream != (ISocketStream *)NULL) { - delete _body_stream; - _body_stream = (ISocketStream *)NULL; - } + reset_body_stream(); _source.clear(); _bio.clear(); _working_get = string(); diff --git a/panda/src/downloader/httpChannel.h b/panda/src/downloader/httpChannel.h index 9067e37d4d..68cf6eaa69 100644 --- a/panda/src/downloader/httpChannel.h +++ b/panda/src/downloader/httpChannel.h @@ -29,7 +29,6 @@ #include "httpEnum.h" #include "urlSpec.h" #include "documentSpec.h" -#include "virtualFile.h" #include "bioPtr.h" #include "bioStreamPtr.h" #include "pmap.h" @@ -38,6 +37,7 @@ #include "config_downloader.h" #include "filename.h" #include "openssl/ssl.h" +#include "typedReferenceCount.h" class Ramfile; class HTTPClient; @@ -57,22 +57,13 @@ class HTTPClient; // requested from the same HTTPChannel until the first // document has been fully retrieved. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS HTTPChannel : public VirtualFile { +class EXPCL_PANDAEXPRESS HTTPChannel : public TypedReferenceCount { private: HTTPChannel(HTTPClient *client); public: virtual ~HTTPChannel(); - virtual VirtualFileSystem *get_file_system() const; - virtual Filename get_filename() const; - - virtual bool is_regular_file() const; - virtual istream *open_read_file(bool auto_unwrap) const; - void close_read_file(istream *stream) const; - - bool will_close_connection() const; - PUBLISHED: // get_status_code() will either return an HTTP-style status code >= // 100 (e.g. 404), or one of the following values. In general, @@ -99,8 +90,8 @@ PUBLISHED: SC_ssl_invalid_server_certificate, SC_ssl_unexpected_server, - // These errors are only generated after a download_to_ram() or - // download_to_file() call has been issued. + // These errors are only generated after a download_to_*() call + // been issued. SC_download_open_error, SC_download_write_error, SC_download_invalid_range, @@ -108,6 +99,7 @@ PUBLISHED: INLINE bool is_valid() const; INLINE bool is_connection_ready() const; + INLINE const URLSpec &get_url() const; INLINE const DocumentSpec &get_document_spec() const; INLINE HTTPEnum::HTTPVersion get_http_version() const; @@ -124,6 +116,7 @@ PUBLISHED: INLINE void set_persistent_connection(bool persistent_connection); INLINE bool get_persistent_connection() const; + bool will_close_connection() const; INLINE void set_allow_proxy(bool allow_proxy); INLINE bool get_allow_proxy() const; @@ -138,6 +131,11 @@ PUBLISHED: INLINE void set_http_timeout(double timeout_seconds); INLINE double get_http_timeout() const; + INLINE void set_skip_body_size(size_t skip_body_size); + INLINE size_t get_skip_body_size() const; + INLINE void set_idle_timeout(double idle_timeout); + INLINE double get_idle_timeout() const; + INLINE void set_download_throttle(bool download_throttle); INLINE bool get_download_throttle() const; @@ -148,7 +146,7 @@ PUBLISHED: INLINE double get_max_updates_per_second() const; INLINE void set_expected_file_size(size_t file_size); - virtual off_t get_file_size() const; + off_t get_file_size() const; INLINE bool is_file_size_known() const; INLINE size_t get_first_byte_requested() const; @@ -183,9 +181,12 @@ PUBLISHED: bool run(); INLINE void begin_connect_to(const DocumentSpec &url); - ISocketStream *read_body(); + ISocketStream *open_read_body(); + void close_read_body(istream *stream) const; + BLOCKING bool download_to_file(const Filename &filename, bool subdocument_resumes = true); BLOCKING bool download_to_ram(Ramfile *ramfile, bool subdocument_resumes = true); + BLOCKING bool download_to_stream(ostream *strm, bool subdocument_resumes = true); SocketStream *get_connection(); INLINE size_t get_bytes_downloaded() const; @@ -194,6 +195,7 @@ PUBLISHED: public: static string downcase(const string &s); + void body_stream_destructs(ISocketStream *stream); private: bool reached_done_state(); @@ -220,6 +222,7 @@ private: bool run_download_to_file(); bool run_download_to_ram(); + bool run_download_to_stream(); void begin_request(HTTPEnum::Method method, const DocumentSpec &url, const string &body, bool nonblocking, @@ -258,6 +261,7 @@ private: void reset_download_to(); void reset_to_new(); + void reset_body_stream(); void close_connection(); static bool more_useful_status_code(int a, int b); @@ -312,6 +316,8 @@ private: bool _proxy_tunnel; double _connect_timeout; double _http_timeout; + size_t _skip_body_size; + double _idle_timeout; bool _blocking_connect; bool _download_throttle; double _max_bytes_per_second; @@ -342,12 +348,14 @@ private: DD_none, DD_file, DD_ram, + DD_stream, }; DownloadDest _download_dest; bool _subdocument_resumes; Filename _download_to_filename; pofstream _download_to_file; Ramfile *_download_to_ramfile; + ostream *_download_to_stream; int _read_index; @@ -406,6 +414,7 @@ private: string _current_field_name; string _current_field_value; ISocketStream *_body_stream; + bool _owns_body_stream; BIO *_sbio; pvector _redirect_trail; int _last_status_code; @@ -420,9 +429,9 @@ public: return _type_handle; } static void init_type() { - VirtualFile::init_type(); + TypedReferenceCount::init_type(); register_type(_type_handle, "HTTPChannel", - VirtualFile::get_class_type()); + TypedReferenceCount::get_class_type()); } private: diff --git a/panda/src/downloader/httpClient.cxx b/panda/src/downloader/httpClient.cxx index bbbcf284c2..73f9cd5daa 100644 --- a/panda/src/downloader/httpClient.cxx +++ b/panda/src/downloader/httpClient.cxx @@ -39,6 +39,7 @@ bool HTTPClient::_ssl_initialized = false; // This is created once and never freed. X509_STORE *HTTPClient::_x509_store = NULL; +PT(HTTPClient) HTTPClient::_global_ptr; //////////////////////////////////////////////////////////////////// // Function: trim_blanks @@ -1080,6 +1081,19 @@ get_header(const URLSpec &url) { return doc; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::get_global_ptr +// Access: Published, Static +// Description: Returns the default global HTTPClient. +//////////////////////////////////////////////////////////////////// +HTTPClient *HTTPClient:: +get_global_ptr() { + if (_global_ptr == NULL) { + _global_ptr = new HTTPClient; + } + return _global_ptr; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPClient::get_ssl_ctx diff --git a/panda/src/downloader/httpClient.h b/panda/src/downloader/httpClient.h index 03fab01829..52d7e6fcbd 100644 --- a/panda/src/downloader/httpClient.h +++ b/panda/src/downloader/httpClient.h @@ -34,6 +34,7 @@ #include "pvector.h" #include "pmap.h" #include "pset.h" +#include "referenceCount.h" #include "openssl/ssl.h" @@ -49,12 +50,18 @@ class HTTPChannel; // Class : HTTPClient // Description : Handles contacting an HTTP server and retrieving a // document. Each HTTPClient object represents a -// separate context; it is up to the programmer whether -// one HTTPClient should be used to retrieve all -// documents, or a separate one should be created each -// time. +// separate context, and stores its own list of cookies, +// passwords, and certificates; however, a given +// HTTPClient is capable of making multiple simultaneous +// requests to the same or different servers. +// +// It is up to the programmer whether one HTTPClient +// should be used to retrieve all documents, or a +// separate one should be created each time. There is a +// default, global HTTPClient available in +// HTTPClient::get_global_ptr(). //////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS HTTPClient { +class EXPCL_PANDAEXPRESS HTTPClient : public ReferenceCount { PUBLISHED: HTTPClient(); HTTPClient(const HTTPClient ©); @@ -128,6 +135,8 @@ PUBLISHED: INLINE static string base64_encode(const string &s); INLINE static string base64_decode(const string &s); + static HTTPClient *get_global_ptr(); + public: SSL_CTX *get_ssl_ctx(); @@ -199,6 +208,9 @@ private: static bool _ssl_initialized; static X509_STORE *_x509_store; + + static PT(HTTPClient) _global_ptr; + friend class HTTPChannel; }; diff --git a/panda/src/downloader/identityStream.I b/panda/src/downloader/identityStream.I index fd51f0811a..dcaef1f8ca 100644 --- a/panda/src/downloader/identityStream.I +++ b/panda/src/downloader/identityStream.I @@ -44,6 +44,6 @@ INLINE IIdentityStream &IIdentityStream:: open(BioStreamPtr *source, HTTPChannel *doc, bool has_content_length, size_t content_length) { clear((ios_iostate)0); - _buf.open_read(source, has_content_length, content_length); + _buf.open_read(source, doc, has_content_length, content_length); return *this; } diff --git a/panda/src/downloader/identityStream.cxx b/panda/src/downloader/identityStream.cxx index f4e88a8ed4..f47d806c80 100644 --- a/panda/src/downloader/identityStream.cxx +++ b/panda/src/downloader/identityStream.cxx @@ -17,6 +17,19 @@ // This module is not compiled if OpenSSL is not available. #ifdef HAVE_OPENSSL +//////////////////////////////////////////////////////////////////// +// Function: IIdentityStream::Destructor +// Access: Published, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +IIdentityStream:: +~IIdentityStream() { + if (_channel != (HTTPChannel *)NULL) { + _channel->body_stream_destructs(this); + _channel = NULL; + } +} + //////////////////////////////////////////////////////////////////// // Function: IIdentityStream::is_closed // Access: Public, Virtual diff --git a/panda/src/downloader/identityStream.h b/panda/src/downloader/identityStream.h index 8a3e4a3396..db95b1dd9f 100644 --- a/panda/src/downloader/identityStream.h +++ b/panda/src/downloader/identityStream.h @@ -48,6 +48,7 @@ public: INLINE IIdentityStream &open(BioStreamPtr *source, HTTPChannel *doc, bool has_content_length, size_t content_length); + virtual ~IIdentityStream(); virtual bool is_closed(); virtual void close(); diff --git a/panda/src/downloader/identityStreamBuf.cxx b/panda/src/downloader/identityStreamBuf.cxx index 21c6973165..3db79f1c81 100644 --- a/panda/src/downloader/identityStreamBuf.cxx +++ b/panda/src/downloader/identityStreamBuf.cxx @@ -31,6 +31,7 @@ IdentityStreamBuf:: IdentityStreamBuf() { _has_content_length = true; _bytes_remaining = 0; + _wanted_nonblocking = false; _read_state = ISocketStream::RS_initial; #ifdef HAVE_IOSTREAM @@ -67,10 +68,11 @@ IdentityStreamBuf:: // from the identity encoding. //////////////////////////////////////////////////////////////////// void IdentityStreamBuf:: -open_read(BioStreamPtr *source, +open_read(BioStreamPtr *source, HTTPChannel *doc, bool has_content_length, size_t content_length) { _source = source; _has_content_length = has_content_length; + _wanted_nonblocking = doc->_wanted_nonblocking; _bytes_remaining = content_length; _read_state = ISocketStream::RS_reading; } @@ -134,6 +136,15 @@ read_chars(char *start, size_t length) { // file. (*_source)->read(start, length); read_count = (*_source)->gcount(); + + if (!_wanted_nonblocking) { + while (read_count == 0 && !(*_source)->is_closed()) { + // Simulate blocking. + thread_yield(); + (*_source)->read(start, length); + read_count = (*_source)->gcount(); + } + } if (read_count == 0) { if ((*_source)->is_closed()) { @@ -144,12 +155,21 @@ read_chars(char *start, size_t length) { } } else { - // Extract some of the bytes remaining in the chunk. + // Extract some of the remaining bytes, but do not read past the + // content_length restriction. if (_bytes_remaining != 0) { length = min(length, _bytes_remaining); (*_source)->read(start, length); read_count = (*_source)->gcount(); + if (!_wanted_nonblocking) { + while (read_count == 0 && !(*_source)->is_closed()) { + // Simulate blocking. + thread_yield(); + (*_source)->read(start, length); + read_count = (*_source)->gcount(); + } + } nassertr(read_count <= _bytes_remaining, 0); _bytes_remaining -= read_count; diff --git a/panda/src/downloader/identityStreamBuf.h b/panda/src/downloader/identityStreamBuf.h index 6fefd67b4b..b546f24e3e 100644 --- a/panda/src/downloader/identityStreamBuf.h +++ b/panda/src/downloader/identityStreamBuf.h @@ -24,6 +24,8 @@ #include "pointerTo.h" #include "socketStream.h" +class HTTPChannel; + //////////////////////////////////////////////////////////////////// // Class : IdentityStreamBuf // Description : The streambuf object that implements @@ -34,7 +36,7 @@ public: IdentityStreamBuf(); virtual ~IdentityStreamBuf(); - void open_read(BioStreamPtr *source, + void open_read(BioStreamPtr *source, HTTPChannel *doc, bool has_content_length, size_t content_length); void close_read(); @@ -50,6 +52,7 @@ private: PT(BioStreamPtr) _source; bool _has_content_length; size_t _bytes_remaining; + bool _wanted_nonblocking; ISocketStream::ReadState _read_state; char *_buffer; diff --git a/panda/src/downloader/socketStream.I b/panda/src/downloader/socketStream.I index c775025c94..4d34ba8a98 100644 --- a/panda/src/downloader/socketStream.I +++ b/panda/src/downloader/socketStream.I @@ -181,6 +181,7 @@ flush() { //////////////////////////////////////////////////////////////////// INLINE ISocketStream:: ISocketStream(streambuf *buf) : istream(buf), SSReader(this) { + _channel = NULL; } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/downloader/socketStream.cxx b/panda/src/downloader/socketStream.cxx index b4e32f6d0b..1e5641566f 100644 --- a/panda/src/downloader/socketStream.cxx +++ b/panda/src/downloader/socketStream.cxx @@ -15,6 +15,7 @@ #include "socketStream.h" #include "datagram.h" #include "datagramIterator.h" +#include "httpChannel.h" #ifdef HAVE_OPENSSL @@ -220,4 +221,31 @@ send_datagram(const Datagram &dg) { return !is_closed(); } +//////////////////////////////////////////////////////////////////// +// Function: ISocketStream::Destructor +// Access: Published, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +ISocketStream:: +~ISocketStream() { + // This should already have been cleared by the subclass destructor. + nassertv(_channel == NULL); +} + + +//////////////////////////////////////////////////////////////////// +// Function: ISocketStream::set_hold_ptr +// Access: Published +// Description: Associates a reference-counting pointer with the +// stream. The stream does nothing with the pointer +// other than hold its reference count for its own +// lifetime (or until set_hold_ptr() is called again). +// +// At the moment, this is used only by VirtualFileHTTP. +//////////////////////////////////////////////////////////////////// +void ISocketStream:: +set_hold_ptr(const TypedReferenceCount *ptr) { + _hold_ptr = ptr; +} + #endif // HAVE_OPENSSL diff --git a/panda/src/downloader/socketStream.h b/panda/src/downloader/socketStream.h index 53bf0ea6db..f897532c0d 100644 --- a/panda/src/downloader/socketStream.h +++ b/panda/src/downloader/socketStream.h @@ -20,6 +20,8 @@ #include "config_express.h" // for collect_tcp #include "datagram.h" #include "pdeque.h" +#include "typedReferenceCount.h" +#include "pointerTo.h" // At the present, this module is not compiled if OpenSSL is not // available, since the only current use for it is to implement @@ -27,6 +29,8 @@ #ifdef HAVE_OPENSSL +class HTTPChannel; + //////////////////////////////////////////////////////////////////// // Class : SSReader // Description : An internal class for reading from a socket stream. @@ -120,6 +124,7 @@ private: class EXPCL_PANDAEXPRESS ISocketStream : public istream, public SSReader { public: INLINE ISocketStream(streambuf *buf); + virtual ~ISocketStream(); PUBLISHED: enum ReadState { @@ -132,6 +137,16 @@ PUBLISHED: virtual bool is_closed() = 0; virtual void close() = 0; virtual ReadState get_read_state() = 0; + + void set_hold_ptr(const TypedReferenceCount *ptr); + +protected: + HTTPChannel *_channel; + +private: + CPT(TypedReferenceCount) _hold_ptr; + + friend class HTTPChannel; }; //////////////////////////////////////////////////////////////////// diff --git a/panda/src/putil/stringStream.I b/panda/src/downloader/stringStream.I similarity index 100% rename from panda/src/putil/stringStream.I rename to panda/src/downloader/stringStream.I diff --git a/panda/src/putil/stringStream.cxx b/panda/src/downloader/stringStream.cxx similarity index 100% rename from panda/src/putil/stringStream.cxx rename to panda/src/downloader/stringStream.cxx diff --git a/panda/src/putil/stringStream.h b/panda/src/downloader/stringStream.h similarity index 95% rename from panda/src/putil/stringStream.h rename to panda/src/downloader/stringStream.h index 388ea3c839..366f0d1394 100644 --- a/panda/src/putil/stringStream.h +++ b/panda/src/downloader/stringStream.h @@ -24,7 +24,7 @@ // data to an internal buffer, which can be retrieved // and/or set as a string. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDA_PUTIL StringStream : public iostream { +class EXPCL_PANDAEXPRESS StringStream : public iostream { PUBLISHED: INLINE StringStream(); INLINE StringStream(const string &source); diff --git a/panda/src/putil/stringStreamBuf.I b/panda/src/downloader/stringStreamBuf.I similarity index 100% rename from panda/src/putil/stringStreamBuf.I rename to panda/src/downloader/stringStreamBuf.I diff --git a/panda/src/putil/stringStreamBuf.cxx b/panda/src/downloader/stringStreamBuf.cxx similarity index 100% rename from panda/src/putil/stringStreamBuf.cxx rename to panda/src/downloader/stringStreamBuf.cxx diff --git a/panda/src/putil/stringStreamBuf.h b/panda/src/downloader/stringStreamBuf.h similarity index 96% rename from panda/src/putil/stringStreamBuf.h rename to panda/src/downloader/stringStreamBuf.h index 9559599841..2fe5936a2f 100644 --- a/panda/src/putil/stringStreamBuf.h +++ b/panda/src/downloader/stringStreamBuf.h @@ -25,7 +25,7 @@ // contents can be appended to or extracted at any time // by application code. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDA_PUTIL StringStreamBuf : public streambuf { +class EXPCL_PANDAEXPRESS StringStreamBuf : public streambuf { public: StringStreamBuf(); virtual ~StringStreamBuf(); diff --git a/panda/src/downloader/virtualFileHTTP.I b/panda/src/downloader/virtualFileHTTP.I new file mode 100644 index 0000000000..88c620f09c --- /dev/null +++ b/panda/src/downloader/virtualFileHTTP.I @@ -0,0 +1,26 @@ +// Filename: virtualFileHTTP.I +// Created by: drose (31Oct08) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::is_implicit_pz_file +// Access: Published +// Description: Returns true if this file is a .pz file that should +// be implicitly decompressed on load, or false if it is +// not a .pz file or if it should not be decompressed. +//////////////////////////////////////////////////////////////////// +INLINE bool VirtualFileHTTP:: +is_implicit_pz_file() const { + return _implicit_pz_file; +} diff --git a/panda/src/downloader/virtualFileHTTP.cxx b/panda/src/downloader/virtualFileHTTP.cxx new file mode 100644 index 0000000000..c11ed2babb --- /dev/null +++ b/panda/src/downloader/virtualFileHTTP.cxx @@ -0,0 +1,278 @@ +// Filename: virtualFileHTTP.cxx +// Created by: drose (31Oct08) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#include "virtualFileHTTP.h" +#include "virtualFileMountHTTP.h" +#include "stringStream.h" +#include "zStream.h" + +TypeHandle VirtualFileHTTP::_type_handle; + + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +VirtualFileHTTP:: +VirtualFileHTTP(VirtualFileMountHTTP *mount, const Filename &local_filename, + bool implicit_pz_file, bool status_only) : + _mount(mount), + _local_filename(local_filename), + _implicit_pz_file(implicit_pz_file), + _status_only(status_only) +{ + URLSpec url(_mount->get_root()); + url.set_path(_mount->get_root().get_path() + _local_filename.c_str()); + _channel = _mount->get_channel(); + if (_status_only) { + _channel->get_header(url); + } else { + _channel->get_document(url); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::Destructor +// Access: Published, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +VirtualFileHTTP:: +~VirtualFileHTTP() { + // Recycle the associated HTTPChannel, so we can use it again later + // without having to close the connection to the server. + _mount->recycle_channel(_channel); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::get_file_system +// Access: Published, Virtual +// Description: Returns the VirtualFileSystem this file is associated +// with. +//////////////////////////////////////////////////////////////////// +VirtualFileSystem *VirtualFileHTTP:: +get_file_system() const { + return _mount->get_file_system(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::get_filename +// Access: Published, Virtual +// Description: Returns the full pathname to this file within the +// virtual file system. +//////////////////////////////////////////////////////////////////// +Filename VirtualFileHTTP:: +get_filename() const { + string mount_point = _mount->get_mount_point(); + if (_local_filename.empty()) { + if (mount_point.empty()) { + return "/"; + } else { + return string("/") + mount_point; + } + + } else { + if (mount_point.empty()) { + return string("/") + _local_filename.get_fullpath(); + } else { + return string("/") + mount_point + string("/") + _local_filename.get_fullpath(); + } + } +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::has_file +// Access: Published, Virtual +// Description: Returns true if this file exists, false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileHTTP:: +has_file() const { + return _channel->is_valid(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::is_directory +// Access: Published, Virtual +// Description: Returns true if this file represents a directory (and +// scan_directory() may be called), false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileHTTP:: +is_directory() const { + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::is_regular_file +// Access: Published, Virtual +// Description: Returns true if this file represents a regular file +// (and read_file() may be called), false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileHTTP:: +is_regular_file() const { + return _channel->is_valid(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::open_read_file +// Access: Published, Virtual +// Description: Opens the file for reading. Returns a newly +// allocated istream on success (which you should +// eventually delete when you are done reading). +// Returns NULL on failure. +// +// If auto_unwrap is true, an explicitly-named .pz file +// is automatically decompressed and the decompressed +// contents are returned. This is different than +// vfs-implicit-pz, which will automatically decompress +// a file if the extension .pz is *not* given. +//////////////////////////////////////////////////////////////////// +istream *VirtualFileHTTP:: +open_read_file(bool auto_unwrap) const { + if (_status_only) { + return NULL; + } + + // We pre-download the file into a StringStream, then return a + // buffer to that. It seems safer, since we can guarantee the file + // comes all at once without timeouts along the way. + StringStream *strstream = new StringStream; + if (!fetch_file(strstream)) { + delete strstream; + return NULL; + } + + return return_file(strstream, auto_unwrap); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::fetch_file +// Access: Private +// Description: Downloads the entire file from the web server into +// the indicated iostream. Returns true on success, +// false on failure. +// +// This seems to be safer than returning the socket +// stream directly, since this way we can better control +// timeouts and other internet hiccups. We can also +// offer seeking on the resulting stream. +//////////////////////////////////////////////////////////////////// +bool VirtualFileHTTP:: +fetch_file(ostream *buffer_stream) const { + _channel->download_to_stream(buffer_stream, false); + if (!_channel->is_download_complete()) { + // Failure to download fully. Try again to download more. + + URLSpec url(_mount->get_root()); + url.set_path(_mount->get_root().get_path() + _local_filename.c_str()); + + size_t bytes_downloaded = _channel->get_bytes_downloaded(); + size_t last_byte = bytes_downloaded; + + while (bytes_downloaded > 0 && !_channel->is_download_complete()) { + _channel->get_subdocument(url, last_byte, 0); + _channel->download_to_stream(buffer_stream, true); + bytes_downloaded = _channel->get_bytes_downloaded(); + last_byte = _channel->get_last_byte_delivered(); + } + } + + return _channel->is_download_complete() && _channel->is_valid(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::return_file +// Access: Private +// Description: After downloading the entire file via fetch_file(), +// rewinds the file stream and returns it as its own +// readable stream. +//////////////////////////////////////////////////////////////////// +istream *VirtualFileHTTP:: +return_file(istream *buffer_stream, bool auto_unwrap) const { + // Will we be automatically unwrapping a .pz file? + bool do_unwrap = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz")); + + istream *result = buffer_stream; +#ifdef HAVE_ZLIB + if (result != (istream *)NULL && do_unwrap) { + // We have to slip in a layer to decompress the file on the fly. + IDecompressStream *wrapper = new IDecompressStream(result, true); + result = wrapper; + } +#endif // HAVE_ZLIB + + return result; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::was_read_successful +// Access: Public +// Description: Call this method after a reading the istream returned +// by open_read_file() to completion. If it returns +// true, the file was read completely and without error; +// if it returns false, there may have been some errors +// or a truncated file read. This is particularly +// likely if the stream is a VirtualFileHTTP. +//////////////////////////////////////////////////////////////////// +bool VirtualFileHTTP:: +was_read_successful() const { + return _channel->is_valid() && _channel->is_download_complete(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::get_file_size +// Access: Published, Virtual +// Description: Returns the current size on disk (or wherever it is) +// of the already-open file. Pass in the stream that +// was returned by open_read_file(); some +// implementations may require this stream to determine +// the size. +//////////////////////////////////////////////////////////////////// +off_t VirtualFileHTTP:: +get_file_size(istream *stream) const { + return _channel->get_file_size(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::get_file_size +// Access: Published, Virtual +// Description: Returns the current size on disk (or wherever it is) +// of the file before it has been opened. +//////////////////////////////////////////////////////////////////// +off_t VirtualFileHTTP:: +get_file_size() const { + return _channel->get_file_size(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileHTTP::get_timestamp +// Access: Published, Virtual +// Description: Returns a time_t value that represents the time the +// file was last modified, to within whatever precision +// the operating system records this information (on a +// Windows95 system, for instance, this may only be +// accurate to within 2 seconds). +// +// If the timestamp cannot be determined, either because +// it is not supported by the operating system or +// because there is some error (such as file not found), +// returns 0. +//////////////////////////////////////////////////////////////////// +time_t VirtualFileHTTP:: +get_timestamp() const { + const DocumentSpec &spec = _channel->get_document_spec(); + if (spec.has_date()) { + return spec.get_date().get_time(); + } + return 0; +} diff --git a/panda/src/downloader/virtualFileHTTP.h b/panda/src/downloader/virtualFileHTTP.h new file mode 100644 index 0000000000..70d335d9bd --- /dev/null +++ b/panda/src/downloader/virtualFileHTTP.h @@ -0,0 +1,89 @@ +// Filename: virtualFileHTTP.h +// Created by: drose (31Oct08) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#ifndef VIRTUALFILEHTTP_H +#define VIRTUALFILEHTTP_H + +#include "pandabase.h" + +#include "virtualFile.h" +#include "httpChannel.h" +#include "urlSpec.h" + +class VirtualFileMountHTTP; + +//////////////////////////////////////////////////////////////////// +// Class : VirtualFileHTTP +// Description : This maps a document retrieved from an HTTPClient +// into the VirtualFileSystem, allowing models etc. to +// be loaded directly from a web page. +//////////////////////////////////////////////////////////////////// +class EXPCL_PANDAEXPRESS VirtualFileHTTP : public VirtualFile { +public: + VirtualFileHTTP(VirtualFileMountHTTP *mount, + const Filename &local_filename, + bool implicit_pz_file, + bool status_only); + virtual ~VirtualFileHTTP(); + + virtual VirtualFileSystem *get_file_system() const; + virtual Filename get_filename() const; + + virtual bool has_file() const; + virtual bool is_directory() const; + virtual bool is_regular_file() const; + INLINE bool is_implicit_pz_file() const; + + virtual istream *open_read_file(bool auto_unwrap) const; + virtual bool was_read_successful() const; + virtual off_t get_file_size(istream *stream) const; + virtual off_t get_file_size() const; + virtual time_t get_timestamp() const; + +private: + bool fetch_file(ostream *buffer_stream) const; + istream *return_file(istream *buffer_stream, bool auto_unwrap) const; + + VirtualFileMountHTTP *_mount; + Filename _local_filename; + bool _implicit_pz_file; + bool _status_only; + URLSpec _url; + PT(HTTPChannel) _channel; + +public: + virtual TypeHandle get_type() const { + return get_class_type(); + } + virtual TypeHandle force_init_type() {init_type(); return get_class_type();} + +PUBLISHED: + static TypeHandle get_class_type() { + return _type_handle; + } + +public: + static void init_type() { + VirtualFile::init_type(); + register_type(_type_handle, "VirtualFileHTTP", + VirtualFile::get_class_type()); + } + +private: + static TypeHandle _type_handle; +}; + +#include "virtualFileHTTP.I" + +#endif diff --git a/panda/src/downloader/virtualFileMountHTTP.I b/panda/src/downloader/virtualFileMountHTTP.I new file mode 100644 index 0000000000..1ee2759a6f --- /dev/null +++ b/panda/src/downloader/virtualFileMountHTTP.I @@ -0,0 +1,36 @@ +// Filename: virtualFileMountHTTP.I +// Created by: drose (30Oct08) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::get_http_client +// Access: Published +// Description: Returns the HTTPClient object that services this +// mount point. +//////////////////////////////////////////////////////////////////// +INLINE HTTPClient *VirtualFileMountHTTP:: +get_http_client() const { + return _http; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::get_root +// Access: Published +// Description: Returns the URL that represents the root of this +// mount point. +//////////////////////////////////////////////////////////////////// +INLINE const URLSpec &VirtualFileMountHTTP:: +get_root() const { + return _root; +} diff --git a/panda/src/downloader/virtualFileMountHTTP.cxx b/panda/src/downloader/virtualFileMountHTTP.cxx new file mode 100644 index 0000000000..0fb463391f --- /dev/null +++ b/panda/src/downloader/virtualFileMountHTTP.cxx @@ -0,0 +1,227 @@ +// Filename: virtualFileMountHTTP.cxx +// Created by: drose (30Oct08) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#include "virtualFileMountHTTP.h" +#include "virtualFileHTTP.h" +#include "virtualFileSystem.h" + +#ifdef HAVE_OPENSSL + +TypeHandle VirtualFileMountHTTP::_type_handle; + + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::Constructor +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +VirtualFileMountHTTP:: +VirtualFileMountHTTP(const URLSpec &root, HTTPClient *http) : + _http(http), + _root(root) +{ + // Make sure the root ends on a slash. The implicit trailing slash + // is a semi-standard internet convention. + string path = _root.get_path(); + if (!path.empty() && path[path.length() - 1] != '/') { + path += '/'; + _root.set_path(path); + } +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::Destructor +// Access: Public, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +VirtualFileMountHTTP:: +~VirtualFileMountHTTP() { +} + + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::has_file +// Access: Public, Virtual +// Description: Returns true if the indicated file exists within the +// mount system. +//////////////////////////////////////////////////////////////////// +bool VirtualFileMountHTTP:: +has_file(const Filename &) const { + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::is_directory +// Access: Public, Virtual +// Description: Returns true if the indicated file exists within the +// mount system and is a directory. +//////////////////////////////////////////////////////////////////// +bool VirtualFileMountHTTP:: +is_directory(const Filename &) const { + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::is_regular_file +// Access: Public, Virtual +// Description: Returns true if the indicated file exists within the +// mount system and is a regular file. +//////////////////////////////////////////////////////////////////// +bool VirtualFileMountHTTP:: +is_regular_file(const Filename &) const { + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::make_virtual_file +// Access: Public, Virtual +// Description: Constructs and returns a new VirtualFile instance +// that corresponds to the indicated filename within +// this mount point. The returned VirtualFile object +// does not imply that the given file actually exists; +// but if the file does exist, then the handle can be +// used to read it. +//////////////////////////////////////////////////////////////////// +PT(VirtualFile) VirtualFileMountHTTP:: +make_virtual_file(const string &local_filename, + const Filename &original_filename, bool implicit_pz_file, + bool status_only) { + PT(VirtualFileHTTP) vfile = + new VirtualFileHTTP(this, local_filename, implicit_pz_file, status_only); + vfile->set_original_filename(original_filename); + + return vfile.p(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::open_read_file +// Access: Public, Virtual +// Description: Opens the file for reading, if it exists. Returns a +// newly allocated istream on success (which you should +// eventually delete when you are done reading). +// Returns NULL on failure. +//////////////////////////////////////////////////////////////////// +istream *VirtualFileMountHTTP:: +open_read_file(const Filename &) const { + return NULL; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::get_file_size +// Access: Published, Virtual +// Description: Returns the current size on disk (or wherever it is) +// of the already-open file. Pass in the stream that +// was returned by open_read_file(); some +// implementations may require this stream to determine +// the size. +//////////////////////////////////////////////////////////////////// +off_t VirtualFileMountHTTP:: +get_file_size(const Filename &, istream *) const { + return 0; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::get_file_size +// Access: Published, Virtual +// Description: Returns the current size on disk (or wherever it is) +// of the file before it has been opened. +//////////////////////////////////////////////////////////////////// +off_t VirtualFileMountHTTP:: +get_file_size(const Filename &) const { + return 0; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::get_timestamp +// Access: Published, Virtual +// Description: Returns a time_t value that represents the time the +// file was last modified, to within whatever precision +// the operating system records this information (on a +// Windows95 system, for instance, this may only be +// accurate to within 2 seconds). +// +// If the timestamp cannot be determined, either because +// it is not supported by the operating system or +// because there is some error (such as file not found), +// returns 0. +//////////////////////////////////////////////////////////////////// +time_t VirtualFileMountHTTP:: +get_timestamp(const Filename &) const { + return 0; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::scan_directory +// Access: Public, Virtual +// Description: Fills the given vector up with the list of filenames +// that are local to this directory, if the filename is +// a directory. Returns true if successful, or false if +// the file is not a directory or cannot be read. +//////////////////////////////////////////////////////////////////// +bool VirtualFileMountHTTP:: +scan_directory(vector_string &, const Filename &) const { + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::output +// Access: Public, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +void VirtualFileMountHTTP:: +output(ostream &out) const { + out << _root; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::get_channel +// Access: Public +// Description: Returns an HTTPChannel object suitable for use for +// extracting a document from the current URL root. +//////////////////////////////////////////////////////////////////// +PT(HTTPChannel) VirtualFileMountHTTP:: +get_channel() { + PT(HTTPChannel) channel; + _channels_lock.acquire(); + + if (!_channels.empty()) { + // If we have some channels sitting around, grab one. Grab the + // one on the end; it was most recently pushed, and therefore most + // likely to be still alive. + channel = _channels.back(); + _channels.pop_back(); + } else { + // If we don't have any channels standing by, make a new one. + channel = _http->make_channel(true); + } + + _channels_lock.release(); + return channel; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountHTTP::recycle_channel +// Access: Public +// Description: Accepts an HTTPChannel that is no longer being used, +// and restores it to standby duty, so that it will be +// returned by a future call to get_channel(). +//////////////////////////////////////////////////////////////////// +void VirtualFileMountHTTP:: +recycle_channel(HTTPChannel *channel) { + _channels_lock.acquire(); + _channels.push_back(channel); + _channels_lock.release(); +} + +#endif // HAVE_OPENSSL diff --git a/panda/src/downloader/virtualFileMountHTTP.h b/panda/src/downloader/virtualFileMountHTTP.h new file mode 100644 index 0000000000..3676b8b41b --- /dev/null +++ b/panda/src/downloader/virtualFileMountHTTP.h @@ -0,0 +1,95 @@ +// Filename: virtualFileMountHTTP.h +// Created by: drose (30Oct08) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) Carnegie Mellon University. All rights reserved. +// +// All use of this software is subject to the terms of the revised BSD +// license. You should have received a copy of this license along +// with this source code in a file named "LICENSE." +// +//////////////////////////////////////////////////////////////////// + +#ifndef VIRTUALFILEMOUNTHTTP_H +#define VIRTUALFILEMOUNTHTTP_H + +#include "pandabase.h" + +#ifdef HAVE_OPENSSL + +#include "virtualFileMount.h" +#include "httpClient.h" +#include "httpChannel.h" +#include "urlSpec.h" +#include "pointerTo.h" +#include "mutexImpl.h" + +//////////////////////////////////////////////////////////////////// +// Class : VirtualFileMountHTTP +// Description : Maps a web page (URL root) into the +// VirtualFileSystem. +//////////////////////////////////////////////////////////////////// +class EXPCL_PANDAEXPRESS VirtualFileMountHTTP : public VirtualFileMount { +PUBLISHED: + VirtualFileMountHTTP(const URLSpec &root, HTTPClient *http = HTTPClient::get_global_ptr()); + virtual ~VirtualFileMountHTTP(); + + INLINE HTTPClient *get_http_client() const; + INLINE const URLSpec &get_root() const; + +public: + virtual PT(VirtualFile) make_virtual_file(const string &local_filename, + const Filename &original_filename, + bool implicit_pz_file, + bool status_only); + + virtual bool has_file(const Filename &file) const; + virtual bool is_directory(const Filename &file) const; + virtual bool is_regular_file(const Filename &file) const; + + virtual istream *open_read_file(const Filename &file) const; + virtual off_t get_file_size(const Filename &file, istream *stream) const; + virtual off_t get_file_size(const Filename &file) const; + virtual time_t get_timestamp(const Filename &file) const; + + virtual bool scan_directory(vector_string &contents, + const Filename &dir) const; + + virtual void output(ostream &out) const; + + PT(HTTPChannel) get_channel(); + void recycle_channel(HTTPChannel *channel); + +private: + PT(HTTPClient) _http; + URLSpec _root; + + MutexImpl _channels_lock; + typedef pvector< PT(HTTPChannel) > Channels; + Channels _channels; + +public: + virtual TypeHandle get_type() const { + return get_class_type(); + } + virtual TypeHandle force_init_type() {init_type(); return get_class_type();} + static TypeHandle get_class_type() { + return _type_handle; + } + static void init_type() { + VirtualFileMount::init_type(); + register_type(_type_handle, "VirtualFileMountHTTP", + VirtualFileMount::get_class_type()); + } + +private: + static TypeHandle _type_handle; +}; + +#include "virtualFileMountHTTP.I" + +#endif // HAVE_OPENSSL + +#endif diff --git a/panda/src/express/virtualFile.cxx b/panda/src/express/virtualFile.cxx index dc3a67ee97..1ca48f3bea 100644 --- a/panda/src/express/virtualFile.cxx +++ b/panda/src/express/virtualFile.cxx @@ -20,6 +20,16 @@ TypeHandle VirtualFile::_type_handle; +//////////////////////////////////////////////////////////////////// +// Function: VirtualFile::has_file +// Access: Published, Virtual +// Description: Returns true if this file exists, false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFile:: +has_file() const { + return false; +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFile::is_directory // Access: Published, Virtual @@ -170,7 +180,7 @@ open_read_file(bool auto_unwrap) const { //////////////////////////////////////////////////////////////////// off_t VirtualFile:: get_file_size(istream *stream) const { - return 0; + return get_file_size(); } //////////////////////////////////////////////////////////////////// @@ -228,6 +238,21 @@ close_read_file(istream *stream) const { } } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFile::was_read_successful +// Access: Public +// Description: Call this method after a reading the istream returned +// by open_read_file() to completion. If it returns +// true, the file was read completely and without error; +// if it returns false, there may have been some errors +// or a truncated file read. This is particularly +// likely if the stream is a VirtualFileHTTP. +//////////////////////////////////////////////////////////////////// +bool VirtualFile:: +was_read_successful() const { + return true; +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFile::read_file // Access: Public diff --git a/panda/src/express/virtualFile.h b/panda/src/express/virtualFile.h index c3255861f0..c5568545b7 100644 --- a/panda/src/express/virtualFile.h +++ b/panda/src/express/virtualFile.h @@ -42,6 +42,7 @@ PUBLISHED: virtual Filename get_filename() const=0; INLINE const Filename &get_original_filename() const; + virtual bool has_file() const; virtual bool is_directory() const; virtual bool is_regular_file() const; @@ -54,6 +55,7 @@ PUBLISHED: BLOCKING INLINE string read_file(bool auto_unwrap) const; BLOCKING virtual istream *open_read_file(bool auto_unwrap) const; BLOCKING void close_read_file(istream *stream) const; + virtual bool was_read_successful() const; BLOCKING virtual off_t get_file_size(istream *stream) const; BLOCKING virtual off_t get_file_size() const; BLOCKING virtual time_t get_timestamp() const; diff --git a/panda/src/express/virtualFileComposite.cxx b/panda/src/express/virtualFileComposite.cxx index 72812c9339..e36d414d01 100644 --- a/panda/src/express/virtualFileComposite.cxx +++ b/panda/src/express/virtualFileComposite.cxx @@ -39,6 +39,16 @@ get_filename() const { return _filename; } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileComposite::has_file +// Access: Published, Virtual +// Description: Returns true if this file exists, false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileComposite:: +has_file() const { + return true; +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFileComposite::is_directory // Access: Published, Virtual diff --git a/panda/src/express/virtualFileComposite.h b/panda/src/express/virtualFileComposite.h index 26cc917f20..91bfd06129 100644 --- a/panda/src/express/virtualFileComposite.h +++ b/panda/src/express/virtualFileComposite.h @@ -36,6 +36,7 @@ public: virtual VirtualFileSystem *get_file_system() const; virtual Filename get_filename() const; + virtual bool has_file() const; virtual bool is_directory() const; protected: diff --git a/panda/src/express/virtualFileMount.I b/panda/src/express/virtualFileMount.I index 59dc01a160..3ee8f7ca5d 100644 --- a/panda/src/express/virtualFileMount.I +++ b/panda/src/express/virtualFileMount.I @@ -19,14 +19,9 @@ // Description: //////////////////////////////////////////////////////////////////// INLINE VirtualFileMount:: -VirtualFileMount(VirtualFileSystem *file_system, - const Filename &physical_filename, - const Filename &mount_point, - int mount_flags) : - _file_system(file_system), - _physical_filename(physical_filename), - _mount_point(mount_point), - _mount_flags(mount_flags) +VirtualFileMount() : + _file_system(NULL), + _mount_flags(0) { } @@ -42,17 +37,6 @@ get_file_system() const { } -//////////////////////////////////////////////////////////////////// -// Function: VirtualFileMount::get_physical_filename -// Access: Public -// Description: Returns the name of the source file on the OS -// filesystem of the directory or file that is mounted. -//////////////////////////////////////////////////////////////////// -INLINE const Filename &VirtualFileMount:: -get_physical_filename() const { - return _physical_filename; -} - //////////////////////////////////////////////////////////////////// // Function: VirtualFileMount::get_mount_point // Access: Public diff --git a/panda/src/express/virtualFileMount.cxx b/panda/src/express/virtualFileMount.cxx index 1ed995d71f..7d4eda99b3 100644 --- a/panda/src/express/virtualFileMount.cxx +++ b/panda/src/express/virtualFileMount.cxx @@ -13,6 +13,7 @@ //////////////////////////////////////////////////////////////////// #include "virtualFileMount.h" +#include "virtualFileSimple.h" TypeHandle VirtualFileMount::_type_handle; @@ -24,6 +25,32 @@ TypeHandle VirtualFileMount::_type_handle; //////////////////////////////////////////////////////////////////// VirtualFileMount:: ~VirtualFileMount() { + nassertv(_file_system == NULL); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMount::make_virtual_file +// Access: Public, Virtual +// Description: Constructs and returns a new VirtualFile instance +// that corresponds to the indicated filename within +// this mount point. The returned VirtualFile object +// does not imply that the given file actually exists; +// but if the file does exist, then the handle can be +// used to read it. +//////////////////////////////////////////////////////////////////// +PT(VirtualFile) VirtualFileMount:: +make_virtual_file(const string &local_filename, + const Filename &original_filename, bool implicit_pz_file, + bool) { + Filename local(local_filename); + if (original_filename.is_text()) { + local.set_text(); + } + PT(VirtualFileSimple) file = + new VirtualFileSimple(this, local, implicit_pz_file); + file->set_original_filename(original_filename); + + return file.p(); } //////////////////////////////////////////////////////////////////// @@ -58,7 +85,7 @@ close_read_file(istream *stream) const { //////////////////////////////////////////////////////////////////// void VirtualFileMount:: output(ostream &out) const { - out << get_physical_filename(); + out << get_type(); } //////////////////////////////////////////////////////////////////// @@ -68,5 +95,5 @@ output(ostream &out) const { //////////////////////////////////////////////////////////////////// void VirtualFileMount:: write(ostream &out) const { - out << get_physical_filename() << " on /" << get_mount_point() << "\n"; + out << *this << " on /" << get_mount_point() << "\n"; } diff --git a/panda/src/express/virtualFileMount.h b/panda/src/express/virtualFileMount.h index e38f6cbfb2..0cb4ec0261 100644 --- a/panda/src/express/virtualFileMount.h +++ b/panda/src/express/virtualFileMount.h @@ -17,9 +17,10 @@ #include "pandabase.h" +#include "virtualFile.h" #include "filename.h" #include "pointerTo.h" -#include "typedObject.h" +#include "typedReferenceCount.h" class VirtualFileSystem; @@ -29,19 +30,21 @@ class VirtualFileSystem; // within a VirtualFileSystem. Normally users don't // need to monkey with this class directly. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS VirtualFileMount : public TypedObject { -public: - INLINE VirtualFileMount(VirtualFileSystem *file_system, - const Filename &physical_filename, - const Filename &mount_point, - int mount_flags); +class EXPCL_PANDAEXPRESS VirtualFileMount : public TypedReferenceCount { +PUBLISHED: + INLINE VirtualFileMount(); virtual ~VirtualFileMount(); INLINE VirtualFileSystem *get_file_system() const; - INLINE const Filename &get_physical_filename() const; INLINE const Filename &get_mount_point() const; INLINE int get_mount_flags() const; +public: + virtual PT(VirtualFile) make_virtual_file(const string &local_filename, + const Filename &original_filename, + bool implicit_pz_file, + bool status_only); + virtual bool has_file(const Filename &file) const=0; virtual bool is_directory(const Filename &file) const=0; virtual bool is_regular_file(const Filename &file) const=0; @@ -55,13 +58,12 @@ public: virtual bool scan_directory(vector_string &contents, const Filename &dir) const=0; - +PUBLISHED: virtual void output(ostream &out) const; virtual void write(ostream &out) const; protected: VirtualFileSystem *_file_system; - Filename _physical_filename; Filename _mount_point; int _mount_flags; @@ -75,13 +77,15 @@ public: return _type_handle; } static void init_type() { - TypedObject::init_type(); + TypedReferenceCount::init_type(); register_type(_type_handle, "VirtualFileMount", - TypedObject::get_class_type()); + TypedReferenceCount::get_class_type()); } private: static TypeHandle _type_handle; + + friend class VirtualFileSystem; }; INLINE ostream &operator << (ostream &out, const VirtualFileMount &mount); diff --git a/panda/src/express/virtualFileMountMultifile.I b/panda/src/express/virtualFileMountMultifile.I index 5894fd89a2..b94724d9f8 100644 --- a/panda/src/express/virtualFileMountMultifile.I +++ b/panda/src/express/virtualFileMountMultifile.I @@ -19,12 +19,7 @@ // Description: //////////////////////////////////////////////////////////////////// INLINE VirtualFileMountMultifile:: -VirtualFileMountMultifile(VirtualFileSystem *file_system, - Multifile *multifile, - const Filename &mount_point, - int mount_flags) : - VirtualFileMount(file_system, multifile->get_multifile_name(), - mount_point, mount_flags), +VirtualFileMountMultifile(Multifile *multifile) : _multifile(multifile) { } diff --git a/panda/src/express/virtualFileMountMultifile.cxx b/panda/src/express/virtualFileMountMultifile.cxx index 56de31cc49..d5078d7738 100644 --- a/panda/src/express/virtualFileMountMultifile.cxx +++ b/panda/src/express/virtualFileMountMultifile.cxx @@ -149,3 +149,13 @@ scan_directory(vector_string &contents, const Filename &dir) const { return _multifile->scan_directory(contents, dir); } + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountMultifile::output +// Access: Public, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +void VirtualFileMountMultifile:: +output(ostream &out) const { + out << _multifile->get_multifile_name(); +} diff --git a/panda/src/express/virtualFileMountMultifile.h b/panda/src/express/virtualFileMountMultifile.h index 296185bf7e..2e7a0933cb 100644 --- a/panda/src/express/virtualFileMountMultifile.h +++ b/panda/src/express/virtualFileMountMultifile.h @@ -27,15 +27,13 @@ // VirtualFileSystem. //////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS VirtualFileMountMultifile : public VirtualFileMount { -public: - INLINE VirtualFileMountMultifile(VirtualFileSystem *file_system, - Multifile *multifile, - const Filename &mount_point, - int mount_flags); +PUBLISHED: + INLINE VirtualFileMountMultifile(Multifile *multifile); virtual ~VirtualFileMountMultifile(); INLINE Multifile *get_multifile() const; +public: virtual bool has_file(const Filename &file) const; virtual bool is_directory(const Filename &file) const; virtual bool is_regular_file(const Filename &file) const; @@ -48,6 +46,7 @@ public: virtual bool scan_directory(vector_string &contents, const Filename &dir) const; + virtual void output(ostream &out) const; private: PT(Multifile) _multifile; diff --git a/panda/src/express/virtualFileMountSystem.I b/panda/src/express/virtualFileMountSystem.I index 5ae4b46acd..7ada742341 100644 --- a/panda/src/express/virtualFileMountSystem.I +++ b/panda/src/express/virtualFileMountSystem.I @@ -19,10 +19,18 @@ // Description: //////////////////////////////////////////////////////////////////// INLINE VirtualFileMountSystem:: -VirtualFileMountSystem(VirtualFileSystem *file_system, - const Filename &physical_filename, - const Filename &mount_point, - int mount_flags) : - VirtualFileMount(file_system, physical_filename, mount_point, mount_flags) +VirtualFileMountSystem(const Filename &physical_filename) : + _physical_filename(physical_filename) { } + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountSystem::get_physical_filename +// Access: Public +// Description: Returns the name of the source file on the OS +// filesystem of the directory or file that is mounted. +//////////////////////////////////////////////////////////////////// +INLINE const Filename &VirtualFileMountSystem:: +get_physical_filename() const { + return _physical_filename; +} diff --git a/panda/src/express/virtualFileMountSystem.cxx b/panda/src/express/virtualFileMountSystem.cxx index 86c68f43d9..bf9b5946da 100644 --- a/panda/src/express/virtualFileMountSystem.cxx +++ b/panda/src/express/virtualFileMountSystem.cxx @@ -195,3 +195,13 @@ scan_directory(vector_string &contents, const Filename &dir) const { return pathname.scan_directory(contents); } + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileMountSystem::output +// Access: Public, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +void VirtualFileMountSystem:: +output(ostream &out) const { + out << get_physical_filename(); +} diff --git a/panda/src/express/virtualFileMountSystem.h b/panda/src/express/virtualFileMountSystem.h index 03b81d0972..63735c8e16 100644 --- a/panda/src/express/virtualFileMountSystem.h +++ b/panda/src/express/virtualFileMountSystem.h @@ -25,13 +25,12 @@ // VirtualFileSystem. //////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS VirtualFileMountSystem : public VirtualFileMount { +PUBLISHED: + INLINE VirtualFileMountSystem(const Filename &physical_filename); + + INLINE const Filename &get_physical_filename() const; + public: - INLINE VirtualFileMountSystem(VirtualFileSystem *file_system, - const Filename &physical_filename, - const Filename &mount_point, - int mount_flags); - - virtual bool has_file(const Filename &file) const; virtual bool is_directory(const Filename &file) const; virtual bool is_regular_file(const Filename &file) const; @@ -44,6 +43,11 @@ public: virtual bool scan_directory(vector_string &contents, const Filename &dir) const; + virtual void output(ostream &out) const; + +private: + Filename _physical_filename; + public: virtual TypeHandle get_type() const { return get_class_type(); diff --git a/panda/src/express/virtualFileSimple.cxx b/panda/src/express/virtualFileSimple.cxx index 82c2ebb8b9..d01477baf8 100644 --- a/panda/src/express/virtualFileSimple.cxx +++ b/panda/src/express/virtualFileSimple.cxx @@ -56,6 +56,16 @@ get_filename() const { } } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileSimple::has_file +// Access: Published, Virtual +// Description: Returns true if this file exists, false otherwise. +//////////////////////////////////////////////////////////////////// +bool VirtualFileSimple:: +has_file() const { + return _mount->has_file(_local_filename); +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFileSimple::is_directory // Access: Published, Virtual diff --git a/panda/src/express/virtualFileSimple.h b/panda/src/express/virtualFileSimple.h index ff5b69018f..3f721551d0 100644 --- a/panda/src/express/virtualFileSimple.h +++ b/panda/src/express/virtualFileSimple.h @@ -35,6 +35,7 @@ public: virtual VirtualFileSystem *get_file_system() const; virtual Filename get_filename() const; + virtual bool has_file() const; virtual bool is_directory() const; virtual bool is_regular_file() const; INLINE bool is_implicit_pz_file() const; diff --git a/panda/src/express/virtualFileSystem.I b/panda/src/express/virtualFileSystem.I index c5a8766785..5f55ca9a3a 100644 --- a/panda/src/express/virtualFileSystem.I +++ b/panda/src/express/virtualFileSystem.I @@ -21,7 +21,8 @@ //////////////////////////////////////////////////////////////////// INLINE bool VirtualFileSystem:: exists(const Filename &filename) const { - return get_file(filename) != (VirtualFile *)NULL; + cerr << "exists: " << filename << "\n"; + return get_file(filename, true) != (VirtualFile *)NULL; } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/express/virtualFileSystem.cxx b/panda/src/express/virtualFileSystem.cxx index 10e93d196e..cf9e1c8e12 100644 --- a/panda/src/express/virtualFileSystem.cxx +++ b/panda/src/express/virtualFileSystem.cxx @@ -55,12 +55,9 @@ VirtualFileSystem:: //////////////////////////////////////////////////////////////////// bool VirtualFileSystem:: mount(Multifile *multifile, const string &mount_point, int flags) { - VirtualFileMountMultifile *mount = - new VirtualFileMountMultifile(this, multifile, - normalize_mount_point(mount_point), - flags); - _mounts.push_back(mount); - return true; + VirtualFileMountMultifile *new_mount = + new VirtualFileMountMultifile(multifile); + return mount(new_mount, mount_point, flags); } //////////////////////////////////////////////////////////////////// @@ -94,16 +91,12 @@ mount(const Filename &physical_filename, const string &mount_point, } if (physical_filename.is_directory()) { - VirtualFileMountSystem *mount = - new VirtualFileMountSystem(this, physical_filename, - normalize_mount_point(mount_point), - flags); - _mounts.push_back(mount); - return true; + VirtualFileMountSystem *new_mount = + new VirtualFileMountSystem(physical_filename); + return mount(new_mount, mount_point, flags); } else { // It's not a directory; it must be a Multifile. PT(Multifile) multifile = new Multifile; - multifile->set_encryption_password(password); // For now these are always opened read only. Maybe later we'll @@ -117,6 +110,24 @@ mount(const Filename &physical_filename, const string &mount_point, } } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileSystem::mount +// Access: Published +// Description: Adds the given VirtualFileMount object to the mount +// list. This is a lower-level function that the other +// flavors of mount(); it requires you to create a +// VirtualFileMount object specifically. +//////////////////////////////////////////////////////////////////// +bool VirtualFileSystem:: +mount(VirtualFileMount *mount, const string &mount_point, int flags) { + nassertr(mount->_file_system == NULL, false); + mount->_file_system = this; + mount->_mount_point = normalize_mount_point(mount_point); + mount->_mount_flags = flags; + _mounts.push_back(mount); + return true; +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFileSystem::unmount // Access: Published @@ -137,7 +148,8 @@ unmount(Multifile *multifile) { DCAST(VirtualFileMountMultifile, mount); if (mmount->get_multifile() == multifile) { // Remove this one. Don't increment wi. - delete mount; + mount->_file_system = NULL; + } else { // Don't remove this one. ++wi; @@ -157,10 +169,9 @@ unmount(Multifile *multifile) { //////////////////////////////////////////////////////////////////// // Function: VirtualFileSystem::unmount // Access: Published -// Description: Unmounts all appearances of the indicated physical -// filename (either a directory name or a Multifile -// name) from the file system. Returns the number of -// appearances unmounted. +// Description: Unmounts all appearances of the indicated directory +// name or multifile name from the file system. Returns +// the number of appearances unmounted. //////////////////////////////////////////////////////////////////// int VirtualFileSystem:: unmount(const Filename &physical_filename) { @@ -170,9 +181,59 @@ unmount(const Filename &physical_filename) { VirtualFileMount *mount = (*ri); (*wi) = mount; - if (mount->get_physical_filename() == physical_filename) { + if (mount->is_exact_type(VirtualFileMountSystem::get_class_type())) { + VirtualFileMountSystem *smount = + DCAST(VirtualFileMountSystem, mount); + if (smount->get_physical_filename() == physical_filename) { + // Remove this one. Don't increment wi. + mount->_file_system = NULL; + + } else { + // Don't remove this one. + ++wi; + } + + } else if (mount->is_exact_type(VirtualFileMountMultifile::get_class_type())) { + VirtualFileMountMultifile *mmount = + DCAST(VirtualFileMountMultifile, mount); + if (mmount->get_multifile()->get_multifile_name() == physical_filename) { + // Remove this one. Don't increment wi. + mount->_file_system = NULL; + + } else { + // Don't remove this one. + ++wi; + } + + } else { + // Don't remove this one. + ++wi; + } + ++ri; + } + + int num_removed = _mounts.end() - wi; + _mounts.erase(wi, _mounts.end()); + return num_removed; +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileSystem::unmount +// Access: Published +// Description: Unmounts the indicated VirtualFileMount object +// from the file system. Returns the number of +// appearances unmounted. +//////////////////////////////////////////////////////////////////// +int VirtualFileSystem:: +unmount(VirtualFileMount *mount) { + Mounts::iterator ri, wi; + wi = ri = _mounts.begin(); + while (ri != _mounts.end()) { + (*wi) = (*ri); + if ((*ri) == mount) { // Remove this one. Don't increment wi. - delete mount; + (*ri)->_file_system = NULL; + } else { // Don't remove this one. ++wi; @@ -203,7 +264,8 @@ unmount_point(const string &mount_point) { if (mount->get_mount_point() == nmp) { // Remove this one. Don't increment wi. - delete mount; + mount->_file_system = NULL; + } else { // Don't remove this one. ++wi; @@ -224,10 +286,9 @@ unmount_point(const string &mount_point) { //////////////////////////////////////////////////////////////////// int VirtualFileSystem:: unmount_all() { - Mounts::iterator mi; - for (mi = _mounts.begin(); mi != _mounts.end(); ++mi) { - VirtualFileMount *mount = (*mi); - delete mount; + Mounts::iterator ri; + for (ri = _mounts.begin(); ri != _mounts.end(); ++ri) { + (*ri)->_file_system = NULL; } int num_removed = _mounts.size(); @@ -235,6 +296,28 @@ unmount_all() { return num_removed; } +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileSystem::get_num_mounts +// Access: Published +// Description: Returns the number of individual mounts in the +// system. +//////////////////////////////////////////////////////////////////// +int VirtualFileSystem:: +get_num_mounts() const { + return _mounts.size(); +} + +//////////////////////////////////////////////////////////////////// +// Function: VirtualFileSystem::get_mount +// Access: Published +// Description: Returns the nth mount in the system. +//////////////////////////////////////////////////////////////////// +VirtualFileMount *VirtualFileSystem:: +get_mount(int n) const { + nassertr(n >= 0 && n < (int)_mounts.size(), NULL); + return _mounts[n]; +} + //////////////////////////////////////////////////////////////////// // Function: VirtualFileSystem::chdir // Access: Published @@ -254,7 +337,7 @@ chdir(const string &new_directory) { return true; } - PT(VirtualFile) file = get_file(new_directory); + PT(VirtualFile) file = get_file(new_directory, true); if (file != (VirtualFile *)NULL && file->is_directory()) { _cwd = file->get_filename(); return true; @@ -278,9 +361,18 @@ get_cwd() const { // Description: Looks up the file by the indicated name in the file // system. Returns a VirtualFile pointer representing // the file if it is found, or NULL if it is not. +// +// If status_only is true, the file will be checked for +// existence and length and so on, but the returned +// file's contents cannot be read. This is an +// optimization which is especially important for +// certain mount types, for instance HTTP, for which +// opening a file to determine its status is +// substantially less expensive than opening it to read +// its contents. //////////////////////////////////////////////////////////////////// PT(VirtualFile) VirtualFileSystem:: -get_file(const Filename &filename) const { +get_file(const Filename &filename, bool status_only) const { nassertr(!filename.empty(), NULL); Filename pathname(filename); if (pathname.is_local()) { @@ -306,26 +398,24 @@ get_file(const Filename &filename) const { if (strpath == mount_point) { // Here's an exact match on the mount point. This filename is // the root directory of this mount object. - if (found_match(found_file, composite_file, mount, "", pathname, - false)) { + if (consider_match(found_file, composite_file, mount, "", pathname, + false, status_only)) { return found_file; } } else if (mount_point.empty()) { // This is the root mount point; all files are in here. - if (mount->has_file(strpath)) { - // Bingo! - if (found_match(found_file, composite_file, mount, strpath, - pathname, false)) { - return found_file; - } -#ifdef HAVE_ZLIB - } else if (vfs_implicit_pz && mount->has_file(strpath_pz)) { - if (found_match(found_file, composite_file, mount, strpath_pz, - pathname, true)) { - return found_file; - } -#endif // HAVE_ZLIB + if (consider_match(found_file, composite_file, mount, strpath, + pathname, false, status_only)) { + return found_file; } +#ifdef HAVE_ZLIB + if (vfs_implicit_pz) { + if (consider_match(found_file, composite_file, mount, strpath_pz, + pathname, true, status_only)) { + return found_file; + } + } +#endif // HAVE_ZLIB } else if (strpath.length() > mount_point.length() && strpath.substr(0, mount_point.length()) == mount_point && @@ -333,21 +423,19 @@ get_file(const Filename &filename) const { // This pathname falls within this mount system. Filename local_filename = strpath.substr(mount_point.length() + 1); Filename local_filename_pz = strpath_pz.substr(mount_point.length() + 1); - if (mount->has_file(local_filename)) { - // Bingo! - if (found_match(found_file, composite_file, mount, local_filename, - pathname, false)) { - return found_file; - } + if (consider_match(found_file, composite_file, mount, local_filename, + pathname, false, status_only)) { + return found_file; + } #ifdef HAVE_ZLIB - } else if (vfs_implicit_pz && mount->has_file(local_filename_pz)) { + if (vfs_implicit_pz) { // Bingo! - if (found_match(found_file, composite_file, mount, local_filename_pz, - pathname, true)) { + if (consider_match(found_file, composite_file, mount, local_filename_pz, + pathname, true, status_only)) { return found_file; } + } #endif // HAVE_ZLIB - } } } return found_file; @@ -362,9 +450,10 @@ get_file(const Filename &filename) const { // found. //////////////////////////////////////////////////////////////////// PT(VirtualFile) VirtualFileSystem:: -find_file(const Filename &filename, const DSearchPath &searchpath) const { +find_file(const Filename &filename, const DSearchPath &searchpath, + bool status_only) const { if (!filename.is_local()) { - return get_file(filename); + return get_file(filename, status_only); } int num_directories = searchpath.get_num_directories(); @@ -378,7 +467,7 @@ find_file(const Filename &filename, const DSearchPath &searchpath) const { // true), we don't prefix another one. match = filename; } - PT(VirtualFile) found_file = get_file(match); + PT(VirtualFile) found_file = get_file(match, status_only); if (found_file != (VirtualFile *)NULL) { return found_file; } @@ -403,7 +492,7 @@ resolve_filename(Filename &filename, PT(VirtualFile) found; if (filename.is_local()) { - found = find_file(filename, searchpath); + found = find_file(filename, searchpath, true); if (found.is_null()) { // We didn't find it with the given extension; can we try the @@ -411,7 +500,7 @@ resolve_filename(Filename &filename, if (filename.get_extension().empty() && !default_extension.empty()) { Filename try_ext = filename; try_ext.set_extension(default_extension); - found = find_file(try_ext, searchpath); + found = find_file(try_ext, searchpath, true); } } } else { @@ -424,7 +513,7 @@ resolve_filename(Filename &filename, if (filename.get_extension().empty() && !default_extension.empty()) { Filename try_ext = filename; try_ext.set_extension(default_extension); - found = get_file(try_ext); + found = get_file(try_ext, true); } } } @@ -605,7 +694,7 @@ get_global_ptr() { //////////////////////////////////////////////////////////////////// istream *VirtualFileSystem:: open_read_file(const Filename &filename, bool auto_unwrap) const { - PT(VirtualFile) file = get_file(filename); + PT(VirtualFile) file = get_file(filename, false); if (file == (VirtualFile *)NULL) { return NULL; } @@ -706,30 +795,34 @@ normalize_mount_point(const string &mount_point) const { } //////////////////////////////////////////////////////////////////// -// Function: VirtualFileSystem::found_match +// Function: VirtualFileSystem::consider_match // Access: Private -// Description: Evaluates one match found during a get_file() -// operation. There may be multiple matches for a -// particular filename due to the ambiguities introduced -// by allowing multiple mount points, so we may have to -// keep searching even after the first match is found. +// Description: Evaluates one possible filename match found during a +// get_file() operation. There may be multiple matches +// for a particular filename due to the ambiguities +// introduced by allowing multiple mount points, so we +// may have to keep searching even after the first match +// is found. // // Returns true if the search should terminate now, or // false if it should keep iterating. //////////////////////////////////////////////////////////////////// bool VirtualFileSystem:: -found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file, - VirtualFileMount *mount, const string &local_filename, - const Filename &original_filename, bool implicit_pz_file) const { +consider_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file, + VirtualFileMount *mount, const string &local_filename, + const Filename &original_filename, bool implicit_pz_file, + bool status_only) const { + PT(VirtualFile) vfile = + mount->make_virtual_file(local_filename, original_filename, false, status_only); + if (!vfile->has_file()) { + // Keep looking. + return false; + } + if (found_file == (VirtualFile *)NULL) { // This was our first match. Save it. - Filename local(local_filename); - if (original_filename.is_text()) { - local.set_text(); - } - found_file = new VirtualFileSimple(mount, local, implicit_pz_file); - found_file->set_original_filename(original_filename); - if (!mount->is_directory(local_filename)) { + found_file = vfile; + if (!found_file->is_directory()) { // If it's not a directory, we're done. return true; } @@ -742,10 +835,11 @@ found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file, } else { // This was our second match. The previous match(es) must // have been directories. - if (!mount->is_directory(local_filename)) { + if (!vfile->is_directory()) { // However, this one isn't a directory. We're done. return true; } + if (!implicit_pz_file) { // At least two directories matched to the same path. We // need a composite directory. @@ -756,8 +850,8 @@ found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file, composite_file->add_component(found_file); found_file = composite_file; } - composite_file->add_component - (new VirtualFileSimple(mount, local_filename, false)); + + composite_file->add_component(vfile); } } diff --git a/panda/src/express/virtualFileSystem.h b/panda/src/express/virtualFileSystem.h index 0ecbf3dbfd..fd3a89ed58 100644 --- a/panda/src/express/virtualFileSystem.h +++ b/panda/src/express/virtualFileSystem.h @@ -18,6 +18,7 @@ #include "pandabase.h" #include "virtualFile.h" +#include "virtualFileMount.h" #include "filename.h" #include "dSearchPath.h" #include "pointerTo.h" @@ -25,7 +26,6 @@ #include "pvector.h" class Multifile; -class VirtualFileMount; class VirtualFileComposite; //////////////////////////////////////////////////////////////////// @@ -45,24 +45,29 @@ PUBLISHED: ~VirtualFileSystem(); enum MountFlags { - MF_owns_pointer = 0x0001, // This flag is no longer used. MF_read_only = 0x0002, }; BLOCKING bool mount(Multifile *multifile, const string &mount_point, int flags); BLOCKING bool mount(const Filename &physical_filename, const string &mount_point, int flags, const string &password = ""); + bool mount(VirtualFileMount *mount, const string &mount_point, int flags); BLOCKING int unmount(Multifile *multifile); BLOCKING int unmount(const Filename &physical_filename); + int unmount(VirtualFileMount *mount); BLOCKING int unmount_point(const string &mount_point); BLOCKING int unmount_all(); + int get_num_mounts() const; + VirtualFileMount *get_mount(int n) const; + BLOCKING bool chdir(const string &new_directory); BLOCKING const Filename &get_cwd() const; - BLOCKING PT(VirtualFile) get_file(const Filename &filename) const; + BLOCKING PT(VirtualFile) get_file(const Filename &filename, bool status_only = false) const; BLOCKING PT(VirtualFile) find_file(const Filename &filename, - const DSearchPath &searchpath) const; + const DSearchPath &searchpath, + bool status_only = false) const; BLOCKING bool resolve_filename(Filename &filename, const DSearchPath &searchpath, const string &default_extension = string()) const; BLOCKING int find_all_files(const Filename &filename, const DSearchPath &searchpath, @@ -91,13 +96,14 @@ public: private: Filename normalize_mount_point(const string &mount_point) const; - bool found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file, - VirtualFileMount *mount, const string &local_filename, - const Filename &original_filename, bool implicit_pz_file) const; + bool consider_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file, + VirtualFileMount *mount, const string &local_filename, + const Filename &original_filename, bool implicit_pz_file, + bool status_only) const; static void parse_option(const string &option, int &flags, string &password); - typedef pvector Mounts; + typedef pvector Mounts; Mounts _mounts; Filename _cwd; diff --git a/panda/src/gobj/texture.h b/panda/src/gobj/texture.h index 35f1610f5a..926d563f1b 100644 --- a/panda/src/gobj/texture.h +++ b/panda/src/gobj/texture.h @@ -618,7 +618,7 @@ protected: Filename _alpha_filename; Filename _fullpath; Filename _alpha_fullpath; - string _texture_pool_key; + Filename _texture_pool_key; // The number of channels of the primary file we use. 1, 2, 3, or 4. int _primary_file_num_channels; diff --git a/panda/src/gobj/texturePool.I b/panda/src/gobj/texturePool.I index dce2a93fa7..f7b6e30e0f 100644 --- a/panda/src/gobj/texturePool.I +++ b/panda/src/gobj/texturePool.I @@ -202,6 +202,18 @@ release_all_textures() { get_global_ptr()->ns_release_all_textures(); } +//////////////////////////////////////////////////////////////////// +// Function: TexturePool::rehash +// Access: Published, Static +// Description: Should be called when the model-path changes, to blow +// away the cache of texture pathnames found along the +// model-path. +//////////////////////////////////////////////////////////////////// +INLINE void TexturePool:: +rehash() { + get_global_ptr()->_relpath_lookup.clear(); +} + //////////////////////////////////////////////////////////////////// // Function: TexturePool::garbage_collect // Access: Published, Static diff --git a/panda/src/gobj/texturePool.cxx b/panda/src/gobj/texturePool.cxx index 869766f1c1..42a7b0ed90 100644 --- a/panda/src/gobj/texturePool.cxx +++ b/panda/src/gobj/texturePool.cxx @@ -24,7 +24,7 @@ #include "texturePoolFilter.h" #include "configVariableList.h" #include "load_dso.h" -#include "lightMutexHolder.h" +#include "mutexHolder.h" TexturePool *TexturePool::_global_ptr; @@ -51,7 +51,7 @@ write(ostream &out) { //////////////////////////////////////////////////////////////////// void TexturePool:: register_texture_type(MakeTextureFunc *func, const string &extensions) { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); vector_string words; extract_words(downcase(extensions), words); @@ -70,7 +70,7 @@ register_texture_type(MakeTextureFunc *func, const string &extensions) { //////////////////////////////////////////////////////////////////// void TexturePool:: register_filter(TexturePoolFilter *filter) { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); gobj_cat.info() << "Registering Texture filter " << *filter << "\n"; @@ -87,7 +87,7 @@ register_filter(TexturePoolFilter *filter) { //////////////////////////////////////////////////////////////////// TexturePool::MakeTextureFunc *TexturePool:: get_texture_type(const string &extension) const { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); string c = downcase(extension); TypeRegistry::const_iterator ti; @@ -139,7 +139,7 @@ make_texture(const string &extension) const { //////////////////////////////////////////////////////////////////// void TexturePool:: write_texture_types(ostream &out, int indent_level) const { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); PNMFileTypeRegistry *pnm_reg = PNMFileTypeRegistry::get_global_ptr(); pnm_reg->write(out, indent_level); @@ -206,17 +206,10 @@ TexturePool() { //////////////////////////////////////////////////////////////////// bool TexturePool:: ns_has_texture(const Filename &orig_filename) { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); - Filename filename(orig_filename); - - if (!_fake_texture_image.empty()) { - filename = _fake_texture_image; - } - - VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); - vfs->resolve_filename(filename, get_texture_path()); - vfs->resolve_filename(filename, get_model_path()); + Filename filename; + resolve_filename(filename, orig_filename); Textures::const_iterator ti; ti = _textures.find(filename); @@ -236,18 +229,11 @@ ns_has_texture(const Filename &orig_filename) { Texture *TexturePool:: ns_load_texture(const Filename &orig_filename, int primary_file_num_channels, bool read_mipmaps, const LoaderOptions &options) { - Filename filename(orig_filename); - - if (!_fake_texture_image.empty()) { - filename = _fake_texture_image; - } - - VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); - vfs->resolve_filename(filename, get_texture_path()) || - vfs->resolve_filename(filename, get_model_path()); + Filename filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); + resolve_filename(filename, orig_filename); Textures::const_iterator ti; ti = _textures.find(filename); if (ti != _textures.end()) { @@ -312,7 +298,7 @@ ns_load_texture(const Filename &orig_filename, int primary_file_num_channels, tex->_texture_pool_key = filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); // Now look again--someone may have just loaded this texture in // another thread. @@ -359,23 +345,18 @@ ns_load_texture(const Filename &orig_filename, int primary_file_num_channels, int alpha_file_channel, bool read_mipmaps, const LoaderOptions &options) { - Filename filename(orig_filename); - Filename alpha_filename(orig_alpha_filename); - if (!_fake_texture_image.empty()) { return ns_load_texture(_fake_texture_image, primary_file_num_channels, read_mipmaps, options); } - VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); - vfs->resolve_filename(filename, get_texture_path()) || - vfs->resolve_filename(filename, get_model_path()); - - vfs->resolve_filename(alpha_filename, get_texture_path()) || - vfs->resolve_filename(alpha_filename, get_model_path()); + Filename filename; + Filename alpha_filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); + resolve_filename(filename, orig_filename); + resolve_filename(alpha_filename, orig_alpha_filename); Textures::const_iterator ti; ti = _textures.find(filename); @@ -444,7 +425,7 @@ ns_load_texture(const Filename &orig_filename, tex->_texture_pool_key = filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); // Now look again. Textures::const_iterator ti; @@ -486,15 +467,13 @@ ns_load_texture(const Filename &orig_filename, Texture *TexturePool:: ns_load_3d_texture(const Filename &filename_pattern, bool read_mipmaps, const LoaderOptions &options) { - Filename filename(filename_pattern); - filename.set_pattern(true); - - VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); - vfs->resolve_filename(filename, get_texture_path()) || - vfs->resolve_filename(filename, get_model_path()); + Filename orig_filename(filename_pattern); + orig_filename.set_pattern(true); + Filename filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); + resolve_filename(filename, orig_filename); Textures::const_iterator ti; ti = _textures.find(filename); @@ -549,7 +528,7 @@ ns_load_3d_texture(const Filename &filename_pattern, tex->_texture_pool_key = filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); // Now look again. Textures::const_iterator ti; @@ -580,15 +559,13 @@ ns_load_3d_texture(const Filename &filename_pattern, Texture *TexturePool:: ns_load_cube_map(const Filename &filename_pattern, bool read_mipmaps, const LoaderOptions &options) { - Filename filename(filename_pattern); - filename.set_pattern(true); - - VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); - vfs->resolve_filename(filename, get_texture_path()) || - vfs->resolve_filename(filename, get_model_path()); + Filename orig_filename(filename_pattern); + orig_filename.set_pattern(true); + Filename filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); + resolve_filename(filename, orig_filename); Textures::const_iterator ti; ti = _textures.find(filename); @@ -643,7 +620,7 @@ ns_load_cube_map(const Filename &filename_pattern, bool read_mipmaps, tex->_texture_pool_key = filename; { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); // Now look again. Textures::const_iterator ti; @@ -673,7 +650,7 @@ ns_load_cube_map(const Filename &filename_pattern, bool read_mipmaps, //////////////////////////////////////////////////////////////////// Texture *TexturePool:: ns_get_normalization_cube_map(int size) { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); if (_normalization_cube_map == (Texture *)NULL) { _normalization_cube_map = new Texture("normalization_cube_map"); @@ -693,7 +670,7 @@ ns_get_normalization_cube_map(int size) { //////////////////////////////////////////////////////////////////// Texture *TexturePool:: ns_get_alpha_scale_map() { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); if (_alpha_scale_map == (Texture *)NULL) { _alpha_scale_map = new Texture("alpha_scale_map"); @@ -711,7 +688,7 @@ ns_get_alpha_scale_map() { void TexturePool:: ns_add_texture(Texture *tex) { PT(Texture) keep = tex; - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); if (!tex->_texture_pool_key.empty()) { ns_release_texture(tex); @@ -734,7 +711,7 @@ ns_add_texture(Texture *tex) { //////////////////////////////////////////////////////////////////// void TexturePool:: ns_release_texture(Texture *tex) { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); if (!tex->_texture_pool_key.empty()) { Textures::iterator ti; @@ -744,6 +721,9 @@ ns_release_texture(Texture *tex) { } tex->_texture_pool_key = string(); } + + // Blow away the cache of resolved relative filenames. + _relpath_lookup.clear(); } //////////////////////////////////////////////////////////////////// @@ -753,7 +733,7 @@ ns_release_texture(Texture *tex) { //////////////////////////////////////////////////////////////////// void TexturePool:: ns_release_all_textures() { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); Textures::iterator ti; for (ti = _textures.begin(); ti != _textures.end(); ++ti) { @@ -763,6 +743,9 @@ ns_release_all_textures() { _textures.clear(); _normalization_cube_map = NULL; + + // Blow away the cache of resolved relative filenames. + _relpath_lookup.clear(); } //////////////////////////////////////////////////////////////////// @@ -772,7 +755,7 @@ ns_release_all_textures() { //////////////////////////////////////////////////////////////////// int TexturePool:: ns_garbage_collect() { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); int num_released = 0; Textures new_set; @@ -814,7 +797,7 @@ ns_garbage_collect() { //////////////////////////////////////////////////////////////////// void TexturePool:: ns_list_contents(ostream &out) const { - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); int total_size; int total_ram_size; @@ -844,6 +827,35 @@ ns_list_contents(ostream &out) const { out << "texture pool size - texture pool ram: " << total_size - total_ram_size << "\n"; } +//////////////////////////////////////////////////////////////////// +// Function: TexturePool::resolve_filename +// Access: Private +// Description: Searches for the indicated filename along the texture +// and model path. If the filename was previously +// searched for, doesn't search again, as an +// optimization. Assumes _lock is held. +//////////////////////////////////////////////////////////////////// +void TexturePool:: +resolve_filename(Filename &new_filename, const Filename &orig_filename) { + if (!_fake_texture_image.empty()) { + new_filename = _fake_texture_image; + return; + } + + RelpathLookup::iterator rpi = _relpath_lookup.find(orig_filename); + if (rpi != _relpath_lookup.end()) { + new_filename = (*rpi).second; + return; + } + + new_filename = orig_filename; + VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); + vfs->resolve_filename(new_filename, get_texture_path()) || + vfs->resolve_filename(new_filename, get_model_path()); + + _relpath_lookup[orig_filename] = new_filename; +} + //////////////////////////////////////////////////////////////////// // Function: TexturePool::try_load_cache // Access: Private @@ -985,7 +997,7 @@ pre_load(const Filename &orig_filename, const Filename &orig_alpha_filename, bool read_mipmaps, const LoaderOptions &options) { PT(Texture) tex; - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); FilterRegistry::iterator fi; for (fi = _filter_registry.begin(); @@ -1011,7 +1023,7 @@ PT(Texture) TexturePool:: post_load(Texture *tex) { PT(Texture) result = tex; - LightMutexHolder holder(_lock); + MutexHolder holder(_lock); FilterRegistry::iterator fi; for (fi = _filter_registry.begin(); diff --git a/panda/src/gobj/texturePool.h b/panda/src/gobj/texturePool.h index 0a8015df13..6ebe62565d 100644 --- a/panda/src/gobj/texturePool.h +++ b/panda/src/gobj/texturePool.h @@ -20,7 +20,7 @@ #include "filename.h" #include "config_gobj.h" #include "loaderOptions.h" -#include "lightMutex.h" +#include "pmutex.h" #include "pmap.h" class TexturePoolFilter; @@ -65,6 +65,7 @@ PUBLISHED: INLINE static void add_texture(Texture *texture); INLINE static void release_texture(Texture *texture); INLINE static void release_all_textures(); + INLINE static void rehash(); INLINE static int garbage_collect(); @@ -118,6 +119,8 @@ private: int ns_garbage_collect(); void ns_list_contents(ostream &out) const; + void resolve_filename(Filename &new_filename, const Filename &orig_filename); + void try_load_cache(PT(Texture) &tex, BamCache *cache, const Filename &filename, PT(BamCacheRecord) &record, bool &compressed_cache_record, @@ -136,9 +139,12 @@ private: static TexturePool *_global_ptr; - LightMutex _lock; - typedef phash_map Textures; - Textures _textures; + Mutex _lock; + typedef pmap Textures; + Textures _textures; // indexed by fullpath + typedef pmap RelpathLookup; + RelpathLookup _relpath_lookup; + string _fake_texture_image; PT(Texture) _normalization_cube_map; diff --git a/panda/src/pipeline/conditionVarSimpleImpl.cxx b/panda/src/pipeline/conditionVarSimpleImpl.cxx index 1f57e805fd..62ac49bd4d 100644 --- a/panda/src/pipeline/conditionVarSimpleImpl.cxx +++ b/panda/src/pipeline/conditionVarSimpleImpl.cxx @@ -51,7 +51,7 @@ wait(double timeout) { // variable semantics, after all). ThreadSimpleManager *manager = ThreadSimpleManager::get_global_ptr(); ThreadSimpleImpl *thread = manager->get_current_thread(); - manager->enqueue_ready(thread); + manager->enqueue_ready(thread, true); manager->next_context(); _mutex.acquire(); @@ -69,7 +69,7 @@ do_notify() { // There had been a thread waiting on this condition variable. // Switch contexts immediately, to make fairness more likely. ThreadSimpleImpl *thread = manager->get_current_thread(); - manager->enqueue_ready(thread); + manager->enqueue_ready(thread, false); manager->next_context(); } } @@ -86,7 +86,7 @@ do_notify_all() { // There had been a thread waiting on this condition variable. // Switch contexts immediately, to make fairness more likely. ThreadSimpleImpl *thread = manager->get_current_thread(); - manager->enqueue_ready(thread); + manager->enqueue_ready(thread, false); manager->next_context(); } } diff --git a/panda/src/pipeline/mutexDebug.cxx b/panda/src/pipeline/mutexDebug.cxx index 09de18ccce..d1bd446fff 100755 --- a/panda/src/pipeline/mutexDebug.cxx +++ b/panda/src/pipeline/mutexDebug.cxx @@ -356,11 +356,13 @@ do_release() { nassertv(_lock_count > 0); } } else { - + + /* if (thread_cat.is_debug()) { thread_cat.debug() << *current_thread << " releasing " << *this << "\n"; } + */ _cvar_impl.notify(); } } diff --git a/panda/src/pipeline/mutexSimpleImpl.cxx b/panda/src/pipeline/mutexSimpleImpl.cxx index 898006e265..62d603141c 100644 --- a/panda/src/pipeline/mutexSimpleImpl.cxx +++ b/panda/src/pipeline/mutexSimpleImpl.cxx @@ -54,7 +54,7 @@ do_release() { // There had been a thread waiting on this mutex. Switch contexts // immediately, to make fairness more likely. ThreadSimpleImpl *thread = manager->get_current_thread(); - manager->enqueue_ready(thread); + manager->enqueue_ready(thread, false); manager->next_context(); } } diff --git a/panda/src/pipeline/threadSimpleImpl.I b/panda/src/pipeline/threadSimpleImpl.I index 812f96fc8f..e0421dbc98 100644 --- a/panda/src/pipeline/threadSimpleImpl.I +++ b/panda/src/pipeline/threadSimpleImpl.I @@ -80,7 +80,7 @@ yield() { ThreadSimpleManager *manager = ThreadSimpleManager::get_global_ptr(); if (manager->is_same_system_thread()) { ThreadSimpleImpl *thread = manager->get_current_thread(); - thread->yield_this(); + thread->yield_this(true); } else { manager->system_yield(); } @@ -109,7 +109,7 @@ INLINE void ThreadSimpleImpl:: consider_yield_this() { double now = _manager->get_current_time(); if (now >= _stop_time) { - yield_this(); + yield_this(false); } } diff --git a/panda/src/pipeline/threadSimpleImpl.cxx b/panda/src/pipeline/threadSimpleImpl.cxx index db79d83a7a..62e29138f2 100644 --- a/panda/src/pipeline/threadSimpleImpl.cxx +++ b/panda/src/pipeline/threadSimpleImpl.cxx @@ -135,7 +135,7 @@ start(ThreadPriority priority, bool joinable) { init_thread_context(&_context, _stack, _stack_size, st_begin_thread, this); - _manager->enqueue_ready(this); + _manager->enqueue_ready(this, false); return true; } @@ -215,8 +215,8 @@ sleep_this(double seconds) { // Description: //////////////////////////////////////////////////////////////////// void ThreadSimpleImpl:: -yield_this() { - _manager->enqueue_ready(this); +yield_this(bool volunteer) { + _manager->enqueue_ready(this, true); _manager->next_context(); } @@ -254,7 +254,7 @@ begin_thread() { // Any threads that were waiting to join with this thread now become ready. JoiningThreads::iterator jti; for (jti = _joining_threads.begin(); jti != _joining_threads.end(); ++jti) { - _manager->enqueue_ready(*jti); + _manager->enqueue_ready(*jti, false); } _joining_threads.clear(); diff --git a/panda/src/pipeline/threadSimpleImpl.h b/panda/src/pipeline/threadSimpleImpl.h index e67d11783b..e96009525c 100644 --- a/panda/src/pipeline/threadSimpleImpl.h +++ b/panda/src/pipeline/threadSimpleImpl.h @@ -81,7 +81,7 @@ public: INLINE static void consider_yield(); void sleep_this(double seconds); - void yield_this(); + void yield_this(bool volunteer); INLINE void consider_yield_this(); INLINE double get_wake_time() const; diff --git a/panda/src/pipeline/threadSimpleManager.cxx b/panda/src/pipeline/threadSimpleManager.cxx index eb6ebd5cfb..31a0c43bfa 100644 --- a/panda/src/pipeline/threadSimpleManager.cxx +++ b/panda/src/pipeline/threadSimpleManager.cxx @@ -88,12 +88,20 @@ ThreadSimpleManager() : // thread will be executed when its turn comes. If the // thread is not the currently executing thread, its // _jmp_context should be filled appropriately. +// +// If volunteer is true, the thread is volunteering to +// sleep before its timeslice has been used up. If +// volunteer is false, the thread would still be running +// if it could. //////////////////////////////////////////////////////////////////// void ThreadSimpleManager:: -enqueue_ready(ThreadSimpleImpl *thread) { +enqueue_ready(ThreadSimpleImpl *thread, bool volunteer) { // We actually add it to _next_ready, so that we can tell when we // have processed every thread in a given epoch. _next_ready.push_back(thread); + if (volunteer) { + ++_num_next_ready_volunteers; + } } //////////////////////////////////////////////////////////////////// @@ -501,7 +509,9 @@ choose_next_context() { while (_ready.empty()) { if (!_next_ready.empty()) { // We've finished an epoch. + bool all_volunteers = (_num_next_ready_volunteers == (int)_next_ready.size()); _ready.swap(_next_ready); + _num_next_ready_volunteers = 0; if (new_epoch && !_tick_records.empty()) { // Pop the oldest timeslice record off when we finish an @@ -519,10 +529,12 @@ choose_next_context() { } else { // Otherwise, we're legitimately at the end of an epoch. - // Yield, to give some time back to the system. - - // On second thought, this might yield too much. - //system_yield(); + if (all_volunteers) { + // All of our non-blocked, non-sleeping threads have + // voluntarily yielded. Therefore, yield the whole + // process. + system_yield(); + } } new_epoch = true; @@ -577,6 +589,10 @@ choose_next_context() { _current_thread = chosen_thread; break; } + + // This thread is not ready to wake up yet. Put it back for next + // epoch. It doesn't count as a volunteer, though--its timeslice + // was used up. _next_ready.push_back(chosen_thread); } diff --git a/panda/src/pipeline/threadSimpleManager.h b/panda/src/pipeline/threadSimpleManager.h index 550685215a..ee7f7f5e54 100644 --- a/panda/src/pipeline/threadSimpleManager.h +++ b/panda/src/pipeline/threadSimpleManager.h @@ -58,7 +58,7 @@ private: ThreadSimpleManager(); public: - void enqueue_ready(ThreadSimpleImpl *thread); + void enqueue_ready(ThreadSimpleImpl *thread, bool volunteer); void enqueue_sleep(ThreadSimpleImpl *thread, double seconds); void enqueue_block(ThreadSimpleImpl *thread, BlockerSimple *blocker); bool unblock_one(BlockerSimple *blocker); @@ -118,6 +118,7 @@ private: // FIFO list of ready threads. FifoThreads _ready; FifoThreads _next_ready; + int _num_next_ready_volunteers; typedef pmap Blocked; Blocked _blocked; diff --git a/panda/src/putil/Sources.pp b/panda/src/putil/Sources.pp index 225998a6cc..cf805e1dd0 100644 --- a/panda/src/putil/Sources.pp +++ b/panda/src/putil/Sources.pp @@ -59,8 +59,6 @@ simpleHashMap.I simpleHashMap.h \ sparseArray.I sparseArray.h \ string_utils.I string_utils.N string_utils.h \ - stringStreamBuf.I stringStreamBuf.h \ - stringStream.I stringStream.h \ timedCycle.I timedCycle.h typedWritable.I \ typedWritable.h typedWritableReferenceCount.I \ typedWritableReferenceCount.h updateSeq.I updateSeq.h \ @@ -107,8 +105,6 @@ simpleHashMap.cxx \ sparseArray.cxx \ string_utils.cxx \ - stringStreamBuf.cxx \ - stringStream.cxx \ timedCycle.cxx typedWritable.cxx \ typedWritableReferenceCount.cxx updateSeq.cxx \ uniqueIdAllocator.cxx \ @@ -167,8 +163,6 @@ simpleHashMap.I simpleHashMap.h \ sparseArray.I sparseArray.h \ string_utils.I string_utils.h \ - stringStreamBuf.I stringStreamBuf.h \ - stringStream.I stringStream.h \ timedCycle.I timedCycle.h typedWritable.I \ typedWritable.h typedWritableReferenceCount.I \ typedWritableReferenceCount.h updateSeq.I updateSeq.h \ diff --git a/panda/src/putil/putil_composite2.cxx b/panda/src/putil/putil_composite2.cxx index f165e5f5b0..d0d864ecb8 100644 --- a/panda/src/putil/putil_composite2.cxx +++ b/panda/src/putil/putil_composite2.cxx @@ -17,8 +17,6 @@ #include "simpleHashMap.cxx" #include "sparseArray.cxx" #include "string_utils.cxx" -#include "stringStreamBuf.cxx" -#include "stringStream.cxx" #include "timedCycle.cxx" #include "typedWritable.cxx" #include "typedWritableReferenceCount.cxx"