From 893078a291e88001763d5dbccaf7ca714e0c686c Mon Sep 17 00:00:00 2001 From: David Rose Date: Wed, 9 Oct 2002 23:45:46 +0000 Subject: [PATCH] robustify http handling --- panda/src/downloader/Sources.pp | 3 + panda/src/downloader/chunkedStream.h | 8 +- panda/src/downloader/chunkedStreamBuf.cxx | 14 +- panda/src/downloader/chunkedStreamBuf.h | 4 +- .../src/downloader/downloader_composite1.cxx | 2 + panda/src/downloader/httpClient.I | 34 - panda/src/downloader/httpClient.cxx | 343 +++---- panda/src/downloader/httpClient.h | 39 +- panda/src/downloader/httpDocument.I | 133 ++- panda/src/downloader/httpDocument.cxx | 840 ++++++++++++++++-- panda/src/downloader/httpDocument.h | 78 +- panda/src/downloader/identityStream.I | 63 ++ panda/src/downloader/identityStream.cxx | 19 + panda/src/downloader/identityStream.h | 60 ++ panda/src/downloader/identityStreamBuf.cxx | 160 ++++ panda/src/downloader/identityStreamBuf.h | 54 ++ panda/src/downloader/urlSpec.I | 30 + panda/src/downloader/urlSpec.h | 4 + 18 files changed, 1609 insertions(+), 279 deletions(-) create mode 100644 panda/src/downloader/identityStream.I create mode 100644 panda/src/downloader/identityStream.cxx create mode 100644 panda/src/downloader/identityStream.h create mode 100644 panda/src/downloader/identityStreamBuf.cxx create mode 100644 panda/src/downloader/identityStreamBuf.h diff --git a/panda/src/downloader/Sources.pp b/panda/src/downloader/Sources.pp index 77cefd1a91..87876f993b 100644 --- a/panda/src/downloader/Sources.pp +++ b/panda/src/downloader/Sources.pp @@ -18,6 +18,7 @@ extractor.h \ httpClient.I httpClient.h \ httpDocument.I httpDocument.h \ + identityStream.I identityStream.h identityStreamBuf.h \ multiplexStream.I multiplexStream.h \ multiplexStreamBuf.I multiplexStreamBuf.h \ urlSpec.I urlSpec.h \ @@ -33,6 +34,7 @@ extractor.cxx \ httpClient.cxx \ httpDocument.cxx \ + identityStream.cxx identityStreamBuf.cxx \ multiplexStream.cxx multiplexStreamBuf.cxx \ urlSpec.cxx \ $[if $[HAVE_NET], downloadDb.cxx downloader.cxx] \ @@ -49,6 +51,7 @@ extractor.h \ httpClient.I httpClient.h \ httpDocument.I httpDocument.h \ + identityStream.I identityStream.h identityStreamBuf.h \ multiplexStream.I multiplexStream.h \ multiplexStreamBuf.I multiplexStreamBuf.I \ patcher.h patcher.I \ diff --git a/panda/src/downloader/chunkedStream.h b/panda/src/downloader/chunkedStream.h index 8df9d87e7b..3cae0aa543 100644 --- a/panda/src/downloader/chunkedStream.h +++ b/panda/src/downloader/chunkedStream.h @@ -33,14 +33,15 @@ class HTTPDocument; // // Seeking is not supported. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS IChunkedStream : public istream { +// No need to export from DLL. +class IChunkedStream : public istream { public: INLINE IChunkedStream(); INLINE IChunkedStream(istream *source, bool owns_source, - HTTPDocument *doc = NULL); + HTTPDocument *doc); INLINE IChunkedStream &open(istream *source, bool owns_source, - HTTPDocument *doc = NULL); + HTTPDocument *doc); INLINE IChunkedStream &close(); private: @@ -52,3 +53,4 @@ private: #endif + diff --git a/panda/src/downloader/chunkedStreamBuf.cxx b/panda/src/downloader/chunkedStreamBuf.cxx index 7b759f0961..557f6b7b8d 100644 --- a/panda/src/downloader/chunkedStreamBuf.cxx +++ b/panda/src/downloader/chunkedStreamBuf.cxx @@ -77,6 +77,7 @@ open_read(istream *source, bool owns_source, HTTPDocument *doc) { _doc = doc; if (_doc != (HTTPDocument *)NULL) { + _read_index = doc->_read_index; _doc->_file_size = 0; // Read a little bit from the file to get the first chunk (and @@ -162,17 +163,20 @@ read_chars(char *start, size_t length) { if (!line.empty() && line[line.length() - 1] == '\r') { line = line.substr(0, line.length() - 1); } - int chunk_size = strtol(line.c_str(), NULL, 16); - if (chunk_size <= 0) { + size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16); + if (chunk_size == 0) { // Last chunk; we're done. _done = true; + if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) { + _doc->_state = HTTPDocument::S_read_body; + } return 0; } - if (_doc != (HTTPDocument *)NULL) { - _doc->_file_size += (size_t)chunk_size; + if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) { + _doc->_file_size += chunk_size; } - _chunk_remaining = (size_t)chunk_size; + _chunk_remaining = chunk_size; return read_chars(start, length); } diff --git a/panda/src/downloader/chunkedStreamBuf.h b/panda/src/downloader/chunkedStreamBuf.h index f77692b6c4..6f7a1c2e93 100644 --- a/panda/src/downloader/chunkedStreamBuf.h +++ b/panda/src/downloader/chunkedStreamBuf.h @@ -28,7 +28,8 @@ // Description : The streambuf object that implements // IChunkedStream. //////////////////////////////////////////////////////////////////// -class EXPCL_PANDAEXPRESS ChunkedStreamBuf : public streambuf { +// No need to export from DLL. +class ChunkedStreamBuf : public streambuf { public: ChunkedStreamBuf(); virtual ~ChunkedStreamBuf(); @@ -48,6 +49,7 @@ private: bool _done; PT(HTTPDocument) _doc; + int _read_index; }; #endif diff --git a/panda/src/downloader/downloader_composite1.cxx b/panda/src/downloader/downloader_composite1.cxx index 10bd93b2ba..f70d25b31b 100644 --- a/panda/src/downloader/downloader_composite1.cxx +++ b/panda/src/downloader/downloader_composite1.cxx @@ -7,6 +7,8 @@ #include "extractor.cxx" #include "httpClient.cxx" #include "httpDocument.cxx" +#include "identityStream.cxx" +#include "identityStreamBuf.cxx" #include "multiplexStream.cxx" #include "multiplexStreamBuf.cxx" #include "urlSpec.cxx" diff --git a/panda/src/downloader/httpClient.I b/panda/src/downloader/httpClient.I index 60f2a9ce0c..fbea8c06a1 100644 --- a/panda/src/downloader/httpClient.I +++ b/panda/src/downloader/httpClient.I @@ -92,37 +92,3 @@ INLINE bool HTTPClient:: get_verify_ssl() const { return _verify_ssl; } - -//////////////////////////////////////////////////////////////////// -// Function: HTTPClient::get_document -// Access: Published -// Description: Opens the named document for reading, or if body is -// nonempty, posts data for a particular URL and -// retrieves the response. Returns a new HTTPDocument -// object whether the document is successfully read or -// not; you can test is_valid() and get_return_code() to -// determine whether the document was retrieved. -//////////////////////////////////////////////////////////////////// -INLINE PT(HTTPDocument) HTTPClient:: -get_document(const URLSpec &url, const string &body) { - const char *method = "GET"; - if (!body.empty()) { - method = "POST"; - } - - return make_request(method, url, body); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPClient::get_header -// Access: Published -// Description: Like get_document(), except only the header -// associated with the file is retrieved. This may be -// used to test for existence of the file; it might also -// return the size of the file (if the server gives us -// this information). -//////////////////////////////////////////////////////////////////// -INLINE PT(HTTPDocument) HTTPClient:: -get_header(const URLSpec &url) { - return make_request("HEAD", url, string()); -} diff --git a/panda/src/downloader/httpClient.cxx b/panda/src/downloader/httpClient.cxx index 8d2b69a77c..0184fecbd2 100644 --- a/panda/src/downloader/httpClient.cxx +++ b/panda/src/downloader/httpClient.cxx @@ -17,6 +17,7 @@ //////////////////////////////////////////////////////////////////// #include "httpClient.h" +#include "httpDocument.h" #include "config_downloader.h" #include "filename.h" #include "config_express.h" @@ -106,6 +107,48 @@ HTTPClient:: clear_expected_servers(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::set_username +// Access: Published +// Description: Specifies the username:password string corresponding +// to a particular server and/or realm, when demanded by +// the server. Either or both of the server or realm +// may be empty; if so, they match anything. Also, the +// server may be set to the special string "*proxy", +// which will match any proxy server. +// +// If the username is set to the empty string, this +// clears the password for the particular server/realm +// pair. +//////////////////////////////////////////////////////////////////// +void HTTPClient:: +set_username(const string &server, const string &realm, const string &username) { + string key = server + ":" + realm; + if (username.empty()) { + _usernames.erase(key); + } else { + _usernames[key] = username; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::get_username +// Access: Published +// Description: Returns the username:password string set for this +// server/realm pair, or empty string if nothing has +// been set. See set_username(). +//////////////////////////////////////////////////////////////////// +string HTTPClient:: +get_username(const string &server, const string &realm) const { + string key = server + ":" + realm; + Usernames::const_iterator ui; + ui = _usernames.find(key); + if (ui != _usernames.end()) { + return (*ui).second; + } + return string(); +} + //////////////////////////////////////////////////////////////////// // Function: HTTPClient::get_http_version_string // Access: Published @@ -120,11 +163,34 @@ get_http_version_string() const { case HV_11: return "HTTP/1.1"; + + case HV_other: + // Report the best we can do. + return "HTTP/1.1"; } return "unknown"; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::parse_http_version_string +// Access: Published +// Description: Matches the string representing a particular HTTP +// version against any of the known versions and returns +// the appropriate enumerated value, or HV_other if the +// version is unknown. +//////////////////////////////////////////////////////////////////// +HTTPClient::HTTPVersion HTTPClient:: +parse_http_version_string(const string &version) { + if (version == "HTTP/1.0") { + return HV_10; + } else if (version == "HTTP/1.1") { + return HV_11; + } else { + return HV_other; + } +} + //////////////////////////////////////////////////////////////////// // Function: HTTPClient::load_certificates // Access: Published @@ -209,6 +275,55 @@ clear_expected_servers() { _expected_servers.clear(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::get_documents +// Access: Published +// Description: Returns a new HTTPDocument object that may be used +// for reading multiple documents using the same +// connection, for greater network efficiency than +// calling HTTPClient::get_document() repeatedly (and +// thus forcing a new connection for each document). +//////////////////////////////////////////////////////////////////// +PT(HTTPDocument) HTTPClient:: +get_documents() { + PT(HTTPDocument) doc = new HTTPDocument(this); + doc->set_persistent_connection(true); + return doc; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::get_document +// Access: Published +// Description: Opens the named document for reading, or if body is +// nonempty, posts data for a particular URL and +// retrieves the response. Returns a new HTTPDocument +// object whether the document is successfully read or +// not; you can test is_valid() and get_return_code() to +// determine whether the document was retrieved. +//////////////////////////////////////////////////////////////////// +PT(HTTPDocument) HTTPClient:: +get_document(const URLSpec &url, const string &body) { + PT(HTTPDocument) doc = new HTTPDocument(this); + doc->get_document(url, body); + return doc; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::get_header +// Access: Published +// Description: Like get_document(), except only the header +// associated with the file is retrieved. This may be +// used to test for existence of the file; it might also +// return the size of the file (if the server gives us +// this information). +//////////////////////////////////////////////////////////////////// +PT(HTTPDocument) HTTPClient:: +get_header(const URLSpec &url) { + PT(HTTPDocument) doc = new HTTPDocument(this); + doc->get_header(url); + return doc; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPClient::make_ctx @@ -376,73 +491,44 @@ load_verify_locations(SSL_CTX *ctx, const Filename &ca_file) { } //////////////////////////////////////////////////////////////////// -// Function: HTTPClient::make_request +// Function: HTTPClient::establish_connection // Access: Private -// Description: Chooses a suitable mechanism to handle this request, -// based on whether it is an http or https request, and -// according to whether we have a proxy in place. -// Issues the request and returns an HTTPDocument that -// represents the results. +// Description: Establishes a connection to the server, using the +// appropriate means. Returns the newly allocated BIO +// representing the connection, or NULL if a connection +// cannot be established. //////////////////////////////////////////////////////////////////// -PT(HTTPDocument) HTTPClient:: -make_request(const string &method, const URLSpec &url, const string &body) { +BIO *HTTPClient:: +establish_connection(const URLSpec &url) { BIO *bio; if (_proxy.empty()) { if (url.get_scheme() == "https") { - bio = get_https(method, url, body); + bio = establish_https(url); } else { - bio = get_http(method, url, body); + bio = establish_http(url); } } else { if (url.get_scheme() == "https") { - bio = get_https_proxy(method, url, body); + bio = establish_https_proxy(url); } else { - bio = get_http_proxy(method, url, body); + bio = establish_http_proxy(url); } } - return new HTTPDocument(bio, true); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPClient::get_https -// Access: Private -// Description: Opens the indicated URL directly as an ordinary http -// document. -//////////////////////////////////////////////////////////////////// -BIO *HTTPClient:: -get_http(const string &method, const URLSpec &url, const string &body) { - ostringstream server; - server << url.get_server() << ":" << url.get_port(); - string server_str = server.str(); - - BIO *bio = BIO_new_connect((char *)server_str.c_str()); - - if (downloader_cat.is_debug()) { - downloader_cat.debug() << "connecting to " << server_str << "\n"; - } - if (BIO_do_connect(bio) <= 0) { - downloader_cat.info() - << "Could not contact server " << server_str << "\n"; -#ifdef REPORT_SSL_ERRORS - ERR_print_errors_fp(stderr); -#endif - return NULL; - } - - send_request(bio, method, url.get_path(), url.get_server(), body); return bio; } //////////////////////////////////////////////////////////////////// -// Function: HTTPClient::get_https +// Function: HTTPClient::establish_http // Access: Private -// Description: Opens the indicated URL directly as an https -// document. +// Description: Establishes a connection to the server directly, +// without using a proxy. Returns the newly allocated +// BIO representing the connection, or NULL if a +// connection cannot be established. //////////////////////////////////////////////////////////////////// BIO *HTTPClient:: -get_https(const string &method, const URLSpec &url, const string &body) { +establish_http(const URLSpec &url) { ostringstream server; server << url.get_server() << ":" << url.get_port(); string server_str = server.str(); @@ -461,22 +547,50 @@ get_https(const string &method, const URLSpec &url, const string &body) { return NULL; } - BIO *sbio = make_https_connection(bio, url); - - if (sbio != (BIO *)NULL) { - send_request(sbio, method, url.get_path(), url.get_server(), body); - } - return sbio; + return bio; } //////////////////////////////////////////////////////////////////// -// Function: HTTPClient::get_http_proxy +// Function: HTTPClient::establish_https // Access: Private -// Description: Opens the indicated URL via the proxy as an ordinary -// http document. +// Description: Establishes a connection to the secure server +// directly, without using a proxy. Returns the newly +// allocated BIO representing the connection, or NULL if +// a connection cannot be established. //////////////////////////////////////////////////////////////////// BIO *HTTPClient:: -get_http_proxy(const string &method, const URLSpec &url, const string &body) { +establish_https(const URLSpec &url) { + ostringstream server; + server << url.get_server() << ":" << url.get_port(); + string server_str = server.str(); + + BIO *bio = BIO_new_connect((char *)server_str.c_str()); + + if (downloader_cat.is_debug()) { + downloader_cat.debug() << "connecting to " << server_str << "\n"; + } + if (BIO_do_connect(bio) <= 0) { + downloader_cat.info() + << "Could not contact server " << server_str << "\n"; +#ifdef REPORT_SSL_ERRORS + ERR_print_errors_fp(stderr); +#endif + return NULL; + } + + return make_https_connection(bio, url); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::establish_http_proxy +// Access: Private +// Description: Establishes a connection to the server through a +// proxy. Returns the newly allocated BIO representing +// the connection, or NULL if a connection cannot be +// established. +//////////////////////////////////////////////////////////////////// +BIO *HTTPClient:: +establish_http_proxy(const URLSpec &url) { ostringstream proxy_server; proxy_server << _proxy.get_server() << ":" << _proxy.get_port(); string proxy_server_str = proxy_server.str(); @@ -496,18 +610,19 @@ get_http_proxy(const string &method, const URLSpec &url, const string &body) { return NULL; } - send_request(bio, method, url, url.get_server(), body); return bio; } //////////////////////////////////////////////////////////////////// -// Function: HTTPClient::get_https_proxy +// Function: HTTPClient::establish_https_proxy // Access: Private -// Description: Opens the indicated URL via the proxy as an https -// document. +// Description: Establishes a connection to the secure server through +// a proxy. Returns the newly allocated BIO +// representing the connection, or NULL if a connection +// cannot be established. //////////////////////////////////////////////////////////////////// BIO *HTTPClient:: -get_https_proxy(const string &method, const URLSpec &url, const string &body) { +establish_https_proxy(const URLSpec &url) { // First, ask the proxy to open a connection for us. ostringstream proxy_server; proxy_server << _proxy.get_server() << ":" << _proxy.get_port(); @@ -528,35 +643,21 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) { return NULL; } - { - ostringstream request; - request - << "CONNECT " << url.get_server() << ":" << url.get_port() - << " " << get_http_version_string() << "\r\n" - << "\r\n"; - string request_str = request.str(); + ostringstream request; + request + << "CONNECT " << url.get_server() << ":" << url.get_port() + << " " << get_http_version_string() << "\r\n"; + string connect_header = request.str(); -#ifndef NDEBUG - if (downloader_cat.is_debug()) { - show_send(request_str); - } -#endif - BIO_puts(bio, request_str.c_str()); - } - - // Create a temporary HTTPDocument to read the response from the - // proxy. + // Create a temporary HTTPDocument to issue the request and read the + // response from the proxy. { - PT(HTTPDocument) doc = new HTTPDocument(bio, false); - if (!doc->is_valid()) { + PT(HTTPDocument) doc = new HTTPDocument(this, bio); + if (!doc->send_request(connect_header, string())) { downloader_cat.info() << "proxy would not open connection to " << url.get_authority() << ": " << doc->get_status_code() << " " << doc->get_status_string() << "\n"; - - if (downloader_cat.is_debug()) { - doc->write_headers(downloader_cat.debug(false)); - } if (!get_verify_ssl()) { // If the proxy refused to open a raw connection for us, see @@ -567,7 +668,7 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) { // us.) if ((doc->get_status_code() / 100) == 4) { BIO_free_all(bio); - return get_http_proxy(method, url, body); + return establish_http_proxy(url); } } return NULL; @@ -581,12 +682,7 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) { // Ok, we now have a connection to our actual server, so start // speaking SSL and then ask for the document we really want. - BIO *sbio = make_https_connection(bio, url); - - if (sbio != (BIO *)NULL) { - send_request(sbio, method, url.get_path(), url.get_server(), body); - } - return sbio; + return make_https_connection(bio, url); } //////////////////////////////////////////////////////////////////// @@ -850,46 +946,6 @@ certificate signing */ -//////////////////////////////////////////////////////////////////// -// Function: HTTPClient::send_request -// Access: Private -// Description: Sends the appropriate GET or POST (or whatever) -// request to the server on the indicated connection. -//////////////////////////////////////////////////////////////////// -void HTTPClient:: -send_request(BIO *bio, const string &method, - const string &path, const string &server, - const string &body) const { - ostringstream request; - - request - << method << " " << path << " " << get_http_version_string() << "\r\n"; - - if (_http_version > HV_10) { - request - << "Host: " << server << "\r\n" - << "Connection: close\r\n"; - } - - if (!body.empty()) { - request - << "Content-type: application/x-www-form-urlencoded\r\n" - << "Content-Length: " << body.length() << "\r\n"; - } - - request - << "\r\n" - << body; - - string request_str = request.str(); -#ifndef NDEBUG - if (downloader_cat.is_debug()) { - show_send(request_str); - } -#endif - BIO_puts(bio, request_str.c_str()); -} - //////////////////////////////////////////////////////////////////// // Function: HTTPClient::verify_server // Access: Private @@ -1089,31 +1145,6 @@ x509_name_subset(X509_NAME *name_a, X509_NAME *name_b) { return true; } -#ifndef NDEBUG -//////////////////////////////////////////////////////////////////// -// Function: HTTPClient::show_send -// Access: Private, Static -// Description: Writes the outgoing message, one line at a time, to -// the debugging log. -//////////////////////////////////////////////////////////////////// -void HTTPClient:: -show_send(const string &message) { - size_t start = 0; - size_t newline = message.find('\n', start); - while (newline != string::npos) { - downloader_cat.debug() - << "send: " << message.substr(start, newline - start + 1); - start = newline + 1; - newline = message.find('\n', start); - } - - if (start < message.length()) { - downloader_cat.debug() - << "send: " << message.substr(start) << " (no newline)\n"; - } -} -#endif // NDEBUG - #if defined(SSL_097) && !defined(NDEBUG) //////////////////////////////////////////////////////////////////// // Function: HTTPClient::ssl_msg_callback diff --git a/panda/src/downloader/httpClient.h b/panda/src/downloader/httpClient.h index 166525d73d..194451bb8d 100644 --- a/panda/src/downloader/httpClient.h +++ b/panda/src/downloader/httpClient.h @@ -29,17 +29,17 @@ #ifdef HAVE_SSL #include "urlSpec.h" -#include "httpDocument.h" #include "pointerTo.h" #include // Windows may define this macro inappropriately. -#ifdef WIN32 +#ifdef X509_NAME #undef X509_NAME #endif class Filename; +class HTTPDocument; //////////////////////////////////////////////////////////////////// // Class : HTTPClient @@ -60,14 +60,19 @@ PUBLISHED: INLINE void set_proxy(const URLSpec &proxy); INLINE const URLSpec &get_proxy() const; + void set_username(const string &server, const string &realm, const string &username); + string get_username(const string &server, const string &realm) const; + enum HTTPVersion { HV_10, // HTTP 1.0 HV_11, // HTTP 1.1 + HV_other, }; INLINE void set_http_version(HTTPVersion version); INLINE HTTPVersion get_http_version() const; string get_http_version_string() const; + static HTTPVersion parse_http_version_string(const string &version); bool load_certificates(const Filename &filename); @@ -77,37 +82,29 @@ PUBLISHED: bool add_expected_server(const string &server_attributes); void clear_expected_servers(); - INLINE PT(HTTPDocument) get_document(const URLSpec &url, - const string &body = string()); - INLINE PT(HTTPDocument) get_header(const URLSpec &url); + PT(HTTPDocument) get_documents(); + PT(HTTPDocument) get_document(const URLSpec &url, + const string &body = string()); + PT(HTTPDocument) get_header(const URLSpec &url); private: void make_ctx(); static void initialize_ssl(); static int load_verify_locations(SSL_CTX *ctx, const Filename &ca_file); - PT(HTTPDocument) make_request(const string &method, const URLSpec &url, - const string &body); - - BIO *get_http(const string &method, const URLSpec &url, const string &body); - BIO *get_https(const string &method, const URLSpec &url, const string &body); - BIO *get_http_proxy(const string &method, const URLSpec &url, const string &body); - BIO *get_https_proxy(const string &method, const URLSpec &url, const string &body); + BIO *establish_connection(const URLSpec &url); + BIO *establish_http(const URLSpec &url); + BIO *establish_https(const URLSpec &url); + BIO *establish_http_proxy(const URLSpec &url); + BIO *establish_https_proxy(const URLSpec &url); BIO *make_https_connection(BIO *bio, const URLSpec &url) const; - void send_request(BIO *bio, const string &method, - const string &path, const string &server, - const string &body) const; bool verify_server(X509_NAME *subject) const; static X509_NAME *parse_x509_name(const string &source); static string get_x509_name_component(X509_NAME *name, int nid); static bool x509_name_subset(X509_NAME *name_a, X509_NAME *name_b); -#ifndef NDEBUG - static void show_send(const string &message); -#endif - #if defined(SSL_097) && !defined(NDEBUG) static void ssl_msg_callback(int write_p, int version, int content_type, const void *buf, size_t len, SSL *ssl, @@ -118,6 +115,9 @@ private: HTTPVersion _http_version; bool _verify_ssl; + typedef pmap Usernames; + Usernames _usernames; + // List of allowable SSL servers to connect to. If the list is // empty, any server is acceptable. typedef pvector ExpectedServers; @@ -127,6 +127,7 @@ private: static bool _ssl_initialized; static X509_STORE *_x509_store; + friend class HTTPDocument; }; #include "httpClient.I" diff --git a/panda/src/downloader/httpDocument.I b/panda/src/downloader/httpDocument.I index fc678632d3..46d5dd015e 100644 --- a/panda/src/downloader/httpDocument.I +++ b/panda/src/downloader/httpDocument.I @@ -29,15 +29,39 @@ is_valid() const { (_status_code / 100) == 2); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_url +// Access: Published +// Description: Returns the URL that was used to retrieve this +// document: whatever URL was last passed to +// get_document() or get_header(). +//////////////////////////////////////////////////////////////////// +INLINE const URLSpec &HTTPDocument:: +get_url() const { + return _url; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPDocument::get_http_version // Access: Published // Description: Returns the HTTP version number returned by the +// server, as one of the HTTPClient enumerated types, +// e.g. HTTPClient::HV_11. +//////////////////////////////////////////////////////////////////// +INLINE HTTPClient::HTTPVersion HTTPDocument:: +get_http_version() const { + return _http_version; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_http_version_string +// Access: Published +// Description: Returns the HTTP version number returned by the // server, formatted as a string, e.g. "HTTP/1.1". //////////////////////////////////////////////////////////////////// INLINE const string &HTTPDocument:: -get_http_version() const { - return _http_version; +get_http_version_string() const { + return _http_version_string; } //////////////////////////////////////////////////////////////////// @@ -65,6 +89,80 @@ get_status_string() const { return _status_string; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_realm +// Access: Published +// Description: If the document failed to connect because of a 401 +// (Authorization required) or 407 (Proxy authorization +// required) error, this method should return the +// "realm" returned by the server in which the requested +// document must be authenticated. This string may be +// presented to the user to request an associated +// username and password (which then should be stored in +// HTTPClient::set_username()). +// +// If the document was retrieved successfully, or failed +// to connect for some other reason, this string will be +// empty. If it is empty even in the presence of a 401 +// or 407 error, then the client code and the server do +// not have a common scheme for exchanging passwords. +//////////////////////////////////////////////////////////////////// +INLINE const string &HTTPDocument:: +get_realm() const { + return _realm; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_redirect +// Access: Published +// Description: If the document failed with a redirect code (300 +// series), this will generally contain the new URL the +// server wants us to try. In many cases, the client +// will automatically follow redirects; if these are +// succesful the client will return a successful code +// and get_redirect() will return empty, but get_url() +// will return the new, redirected URL. +//////////////////////////////////////////////////////////////////// +INLINE const URLSpec &HTTPDocument:: +get_redirect() const { + return _redirect; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::set_persistent_connection +// Access: Published +// Description: Indicates whether the HTTPDocument should try to keep +// the connection to the server open and reuse that +// connection for multiple documents, or whether it +// should close the connection and open a new one for +// each request. Set this true to keep the connections +// around when possible, false to recycle them. +// +// It makes most sense to set this false when the +// HTTPDocument will be used only once to retrieve a +// single document, true when you will be using the +// HTTPDocument interface to retrieve multiple +// documents. +//////////////////////////////////////////////////////////////////// +INLINE void HTTPDocument:: +set_persistent_connection(bool persistent_connection) { + _persistent_connection = persistent_connection; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_persistent_connection +// Access: Published +// Description: Returns whether the HTTPDocument should try to keep +// the connection to the server open and reuse that +// connection for multiple documents, or whether it +// should close the connection and open a new one for +// each request. See set_persistent_connection(). +//////////////////////////////////////////////////////////////////// +INLINE bool HTTPDocument:: +get_persistent_connection() const { + return _persistent_connection; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPDocument::get_file_size // Access: Published @@ -83,3 +181,34 @@ INLINE size_t HTTPDocument:: get_file_size() const { return _file_size; } + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_document +// Access: Published +// Description: Opens the named document for reading, or if body is +// nonempty, posts data for a particular URL and +// retrieves the response. +//////////////////////////////////////////////////////////////////// +INLINE bool HTTPDocument:: +get_document(const URLSpec &url, const string &body) { + const char *method = "GET"; + if (!body.empty()) { + method = "POST"; + } + + return send_request(method, url, body); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_header +// Access: Published +// Description: Like get_document(), except only the header +// associated with the file is retrieved. This may be +// used to test for existence of the file; it might also +// return the size of the file (if the server gives us +// this information). +//////////////////////////////////////////////////////////////////// +INLINE bool HTTPDocument:: +get_header(const URLSpec &url) { + return send_request("HEAD", url, string()); +} diff --git a/panda/src/downloader/httpDocument.cxx b/panda/src/downloader/httpDocument.cxx index 56f30e17d8..49b13830c2 100644 --- a/panda/src/downloader/httpDocument.cxx +++ b/panda/src/downloader/httpDocument.cxx @@ -17,32 +17,148 @@ //////////////////////////////////////////////////////////////////// #include "httpDocument.h" +#include "httpClient.h" #include "bioStream.h" #include "chunkedStream.h" +#include "identityStream.h" #ifdef HAVE_SSL TypeHandle HTTPDocument::_type_handle; +static const char base64_table[64] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', +}; + //////////////////////////////////////////////////////////////////// // Function: HTTPDocument::Constructor -// Access: Public +// Access: Private // Description: //////////////////////////////////////////////////////////////////// HTTPDocument:: -HTTPDocument(BIO *bio, bool owns_bio) { +HTTPDocument(HTTPClient *client, BIO *bio) : + _client(client), + _bio(bio) +{ + _owns_bio = false; + _source = (IBioStream *)NULL; + _persistent_connection = false; + _state = S_new; + _read_index = 0; _file_size = 0; + _status_code = 0; + _status_string = "No connection"; + _proxy = _client->get_proxy(); + _http_version = _client->get_http_version(); + _http_version_string = _client->get_http_version_string(); +} - if (bio != (BIO *)NULL) { - _source = new IBioStream(bio, owns_bio); - read_headers(); - determine_content_length(); +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::send_request +// Access: Private +// Description: This is normally called immediately after +// construction to send the request to the server and +// read the response. It can't be called as part of the +// constructor because it may involve an up-and-down +// change in the reference count of the HTTPDocument +// object, which would inadvertently cause the object to +// be deleted if it hasn't returned from its constructor +// yet! +//////////////////////////////////////////////////////////////////// +bool HTTPDocument:: +send_request(const string &method, const URLSpec &url, const string &body) { + // Let's call this before we call make_header(), so we'll get the + // right HTTP version and proxy information etc. + set_url(url); + prepare_for_next(); - } else { - _source = (IBioStream *)NULL; - _status_code = 0; - _status_string = "No connection"; + string header; + make_header(header, method, url, body); + send_request(header, body); + + if ((get_status_code() / 100) == 3 && get_status_code() != 305) { + // Redirect. Should we handle it automatically? + if (!get_redirect().empty() && (method == "GET" || method == "HEAD")) { + // Sure! + pset already_seen; + bool keep_going; + do { + keep_going = false; + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "following redirect to " << get_redirect() << "\n"; + } + URLSpec new_url = get_redirect(); + if (already_seen.insert(new_url).second) { + if (url.has_username()) { + new_url.set_username(url.get_username()); + } + set_url(new_url); + if (prepare_for_next()) { + make_header(header, method, new_url, body); + send_request(header, body); + keep_going = + ((get_status_code() / 100) == 3 && get_status_code() != 305); + } + } + } while (keep_going); + } } + + return is_valid(); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::send_request +// Access: Private +// Description: This is a lower-level interface than the above +// send_request(); it accepts a header and body string +// that have already been defined. +//////////////////////////////////////////////////////////////////// +bool HTTPDocument:: +send_request(const string &header, const string &body) { + if (prepare_for_next()) { + issue_request(header, body); + + if (get_status_code() == 407 && !_proxy.empty()) { + // 407: not authorized to proxy. Try to get the authorization. + string authenticate_request = get_header_value("Proxy-Authenticate"); + string authorization; + if (get_authorization(authorization, authenticate_request, _proxy, true)) { + string new_header = header; + new_header += "Proxy-Authorization: "; + new_header += authorization; + new_header += "\r\n"; + if (prepare_for_next()) { + issue_request(new_header, body); + } + } + } + + if (get_status_code() == 401) { + // 401: not authorized to remote server. Try to get the authorization. + string authenticate_request = get_header_value("WWW-Authenticate"); + string authorization; + if (get_authorization(authorization, authenticate_request, _url, false)) { + string new_header = header; + new_header += "Authorization: "; + new_header += authorization; + new_header += "\r\n"; + if (prepare_for_next()) { + issue_request(new_header, body); + } + } + } + } + + return is_valid(); } //////////////////////////////////////////////////////////////////// @@ -52,10 +168,7 @@ HTTPDocument(BIO *bio, bool owns_bio) { //////////////////////////////////////////////////////////////////// HTTPDocument:: ~HTTPDocument() { - if (_source != (IBioStream *)NULL) { - delete _source; - _source = (IBioStream *)NULL; - } + free_bio(); } //////////////////////////////////////////////////////////////////// @@ -91,6 +204,31 @@ is_regular_file() const { return is_valid(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::will_close_connection +// Access: Public +// Description: Returns true if the server has indicated it will +// close the connection after this document has been +// read, or false if it will remain open (and future +// documents may be requested on the same connection). +//////////////////////////////////////////////////////////////////// +bool HTTPDocument:: +will_close_connection() const { + if (get_http_version() == HTTPClient::HV_10) { + // HTTP 1.0 always closes. + return true; + } + + string connection = get_header_value("Connection"); + if (downcase(connection) == "close") { + // The server says it will close. + return true; + } + + // Assume the serve will keep it open. + return false; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPDocument::open_read_file // Access: Public, Virtual @@ -102,24 +240,7 @@ is_regular_file() const { //////////////////////////////////////////////////////////////////// istream *HTTPDocument:: open_read_file() const { - if (_source == (IBioStream *)NULL) { - return NULL; - } - - string transfer_coding = get_header_value("Transfer-Encoding"); - for (string::iterator si = transfer_coding.begin(); - si != transfer_coding.end(); - ++si) { - (*si) = tolower(*si); - } - - istream *result = _source; - if (transfer_coding == "chunked") { - result = new IChunkedStream(_source, true, (HTTPDocument *)this); - } - - ((HTTPDocument *)this)->_source = (IBioStream *)NULL; - return result; + return ((HTTPDocument *)this)->read_body(true); } //////////////////////////////////////////////////////////////////// @@ -131,7 +252,7 @@ open_read_file() const { //////////////////////////////////////////////////////////////////// string HTTPDocument:: get_header_value(const string &key) const { - Headers::const_iterator hi = _headers.find(key); + Headers::const_iterator hi = _headers.find(downcase(key)); if (hi != _headers.end()) { return (*hi).second; } @@ -153,15 +274,123 @@ write_headers(ostream &out) const { } //////////////////////////////////////////////////////////////////// -// Function: HTTPDocument::read_headers +// Function: HTTPDocument::make_header +// Access: Private +// Description: Formats the appropriate GET or POST (or whatever) +// request to send to the server. Also saves the +// indicated url. +//////////////////////////////////////////////////////////////////// +void HTTPDocument:: +make_header(string &header, const string &method, + const URLSpec &url, const string &body) { + set_url(url); + _method = method; + + string path; + if (_proxy.empty()) { + path = _url.get_path(); + } else { + URLSpec url_no_username = _url; + url_no_username.set_username(string()); + path = url_no_username.get_url(); + } + + ostringstream stream; + + stream + << method << " " << path << " " << get_http_version_string() << "\r\n"; + + if (_http_version > HTTPClient::HV_10) { + stream + << "Host: " << _url.get_server() << "\r\n"; + if (!get_persistent_connection()) { + stream + << "Connection: close\r\n"; + } + } + + if (!body.empty()) { + stream + << "Content-Type: application/x-www-form-urlencoded\r\n" + << "Content-Length: " << body.length() << "\r\n"; + } + + header = stream.str(); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::set_url +// Access: Private +// Description: Specifies the document's URL before attempting a +// connection. This controls the name of the server to +// be contacted, etc. +//////////////////////////////////////////////////////////////////// +void HTTPDocument:: +set_url(const URLSpec &url) { + // If we change between http and https, we have to reset the + // connection regardless of proxy. Otherwise, we have to drop the + // connection if the server or port changes, unless we're + // communicating through a proxy. + + if (url.get_scheme() != _url.get_scheme() || + (_proxy.empty() && (url.get_server() != _url.get_server() || + url.get_port() != _url.get_port()))) { + free_bio(); + } + _url = url; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::issue_request +// Access: Private +// Description: Issues the request to the HTTP server and waits for a +// response. +//////////////////////////////////////////////////////////////////// +void HTTPDocument:: +issue_request(const string &header, const string &body) { + if (_bio != (BIO *)NULL) { + string request = header; + request += "\r\n"; + request += body; +#ifndef NDEBUG + if (downloader_cat.is_spam()) { + show_send(request); + } +#endif + BIO_puts(_bio, request.c_str()); + read_http_response(); + + if (_source->eof() || _source->fail()) { + if (downloader_cat.is_debug()) { + downloader_cat.debug() + << "Whoops, socket closed.\n"; + free_bio(); + if (prepare_for_next()) { +#ifndef NDEBUG + if (downloader_cat.is_spam()) { + show_send(request); + } +#endif + BIO_puts(_bio, request.c_str()); + read_http_response(); + } + } + } + } +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::read_http_response // Access: Private // Description: Reads all of the responses from the server up until // the first blank line, and stores the list of header // key:value pairs so retrieved. //////////////////////////////////////////////////////////////////// void HTTPDocument:: -read_headers() { +read_http_response() { nassertv(_source != (IBioStream *)NULL); + _headers.clear(); + _realm = string(); // The first line back should include the HTTP version and the // result code. @@ -170,9 +399,10 @@ read_headers() { if (!line.empty() && line[line.length() - 1] == '\r') { line = line.substr(0, line.length() - 1); } - if (downloader_cat.is_debug()) { - downloader_cat.debug() << "recv: " << line << "\n"; + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "recv: " << line << "\n"; } + if (!(*_source) || line.length() < 5 || line.substr(0, 5) != "HTTP/") { // Not an HTTP response. _status_code = 0; @@ -185,7 +415,8 @@ read_headers() { while (p < line.length() && !isspace(line[p])) { p++; } - _http_version = line.substr(0, p); + _http_version_string = line.substr(0, p); + _http_version = HTTPClient::parse_http_version_string(_http_version_string); while (p < line.length() && isspace(line[p])) { p++; @@ -211,9 +442,10 @@ read_headers() { if (!line.empty() && line[line.length() - 1] == '\r') { line = line.substr(0, line.length() - 1); } - if (downloader_cat.is_debug()) { - downloader_cat.debug() << "recv: " << line << "\n"; + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "recv: " << line << "\n"; } + while (!_source->eof() && !_source->fail() && !line.empty()) { if (isspace(line[0])) { // If the line begins with a space, that continues the previous @@ -228,13 +460,13 @@ read_headers() { // If the line does not begin with a space, that defines a new // field. if (!field_name.empty()) { - _headers[field_name] = field_value; + store_header_field(field_name, field_value); field_value = string(); } size_t colon = line.find(':'); if (colon != string::npos) { - field_name = line.substr(0, colon); + field_name = downcase(line.substr(0, colon)); p = colon + 1; while (p < line.length() && isspace(line[p])) { p++; @@ -247,31 +479,531 @@ read_headers() { if (!line.empty() && line[line.length() - 1] == '\r') { line = line.substr(0, line.length() - 1); } - if (downloader_cat.is_debug()) { - downloader_cat.debug() << "recv: " << line << "\n"; + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "recv: " << line << "\n"; } } if (!field_name.empty()) { - _headers[field_name] = field_value; + store_header_field(field_name, field_value); field_value = string(); } // A blank line terminates the headers. -} + _state = S_read_header; -//////////////////////////////////////////////////////////////////// -// Function: HTTPDocument::determine_content_length -// Access: Private -// Description: Determines the file size based on the Content-Length -// field if it has been supplied. -//////////////////////////////////////////////////////////////////// -void HTTPDocument:: -determine_content_length() { + if (get_status_code() / 100 == 1 || + get_status_code() == 204 || + get_status_code() == 304 || + _method == "HEAD") { + // These status codes, or method HEAD, indicate we have no body. + // Therefore, we have already read the (nonexistent) body. + _state = S_read_trailer; + } + + _file_size = 0; string content_length = get_header_value("Content-Length"); if (!content_length.empty()) { _file_size = atoi(content_length.c_str()); } + _redirect = get_header_value("Location"); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::store_header_field +// Access: Private +// Description: Stores a single name: value pair in the header list, +// or appends the value to the end of the existing +// value, if the header has been repeated. +//////////////////////////////////////////////////////////////////// +void HTTPDocument:: +store_header_field(const string &field_name, const string &field_value) { + pair insert_result = + _headers.insert(Headers::value_type(field_name, field_value)); + + if (!insert_result.second) { + // It didn't insert; thus, the field already existed. Append the + // new value. + Headers::iterator hi = insert_result.first; + (*hi).second += ", "; + (*hi).second += field_value; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_authorization +// Access: Private +// Description: Looks for a username:password to satisfy the given +// authenticate_request string from the server or proxy. +// If found, fills in authorization and returns true; +// otherwise, returns false. +//////////////////////////////////////////////////////////////////// +bool HTTPDocument:: +get_authorization(string &authorization, const string &authenticate_request, + const URLSpec &url, bool is_proxy) { + AuthenticationSchemes schemes; + parse_authentication_schemes(schemes, authenticate_request); + + AuthenticationSchemes::iterator si; + si = schemes.find("basic"); + if (si != schemes.end()) { + return get_basic_authorization(authorization, (*si).second, url, is_proxy); + } + + downloader_cat.warning() + << "Don't know how to use any of the server's available authorization schemes:\n"; + for (si = schemes.begin(); si != schemes.end(); ++si) { + downloader_cat.warning() << (*si).first << "\n"; + } + + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::downcase +// Access: Private, Static +// Description: Returns the input string with all uppercase letters +// converted to lowercase. +//////////////////////////////////////////////////////////////////// +string HTTPDocument:: +downcase(const string &s) { + string result; + result.reserve(s.size()); + string::const_iterator p; + for (p = s.begin(); p != s.end(); ++p) { + result += tolower(*p); + } + return result; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::base64_encode +// Access: Private, Static +// Description: Returns the input string encoded using base64. No +// respect is paid to maintaining a 76-char line length. +//////////////////////////////////////////////////////////////////// +string HTTPDocument:: +base64_encode(const string &s) { + // Collect the string 3 bytes at a time into 24-bit words, then + // output each word using 4 bytes. + size_t num_words = (s.size() + 2) / 3; + string result; + result.reserve(num_words * 4); + size_t p; + for (p = 0; p + 2 < s.size(); p += 3) { + unsigned int word = + ((unsigned)s[p] << 16) | + ((unsigned)s[p + 1] << 8) | + ((unsigned)s[p + 2]); + result += base64_table[(word >> 18) & 0x3f]; + result += base64_table[(word >> 12) & 0x3f]; + result += base64_table[(word >> 6) & 0x3f]; + result += base64_table[(word) & 0x3f]; + } + // What's left over? + if (p < s.size()) { + unsigned int word = ((unsigned)s[p] << 16); + p++; + if (p < s.size()) { + word |= ((unsigned)s[p] << 8); + p++; + nassertr(p == s.size(), result); + + result += base64_table[(word >> 18) & 0x3f]; + result += base64_table[(word >> 12) & 0x3f]; + result += base64_table[(word >> 6) & 0x3f]; + result += '='; + } else { + result += base64_table[(word >> 18) & 0x3f]; + result += base64_table[(word >> 12) & 0x3f]; + result += '='; + result += '='; + } + } + + return result; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::scan_quoted_or_unquoted_string +// Access: Private, Static +// Description: Scans the string source beginning at character +// position start, to identify either the +// (spaced-delimited) unquoted string there, or the +// (quote-delimited) quoted string. In either case, +// fills the string found into result, and returns the +// next character position after the string (or after +// its closing quote mark). +//////////////////////////////////////////////////////////////////// +size_t HTTPDocument:: +scan_quoted_or_unquoted_string(string &result, const string &source, + size_t start) { + result = string(); + + if (start < source.length()) { + if (source[start] == '"') { + // Quoted string. + size_t p = start + 1; + while (p < source.length() && source[p] != '"') { + if (source[p] == '\\') { + // Backslash escapes. + ++p; + if (p < source.length()) { + result += source[p]; + ++p; + } + } else { + result += source[p]; + ++p; + } + } + if (p < source.length()) { + ++p; + } + return p; + } + + // Unquoted string. + size_t p = start; + while (p < source.length() && source[p] != ',' && !isspace(source[p])) { + result += source[p]; + ++p; + } + + return p; + } + + // Empty string. + return start; +} + +#ifndef NDEBUG +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::show_send +// Access: Private, Static +// Description: Writes the outgoing message, one line at a time, to +// the debugging log. +//////////////////////////////////////////////////////////////////// +void HTTPDocument:: +show_send(const string &message) { + size_t start = 0; + size_t newline = message.find('\n', start); + while (newline != string::npos) { + downloader_cat.spam() + << "send: " << message.substr(start, newline - start + 1); + start = newline + 1; + newline = message.find('\n', start); + } + + if (start < message.length()) { + downloader_cat.spam() + << "send: " << message.substr(start) << " (no newline)\n"; + } +} +#endif // NDEBUG + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::parse_authentication_schemes +// Access: Private, Static +// Description: Decodes the text following a WWW-Authenticate: or +// Proxy-Authenticate: header field. +//////////////////////////////////////////////////////////////////// +void HTTPDocument:: +parse_authentication_schemes(HTTPDocument::AuthenticationSchemes &schemes, + const string &field_value) { + // This string will consist of one or more records of the form: + // + // scheme token=value[,token=value[,...]] + // + // If there are multiple records, they will be comma-delimited, + // which makes parsing just a bit tricky. + + // Start by skipping initial whitespace. + size_t p = 0; + while (p < field_value.length() && isspace(field_value[p])) { + ++p; + } + + if (p < field_value.length()) { + size_t q = p; + while (q < field_value.length() && !isspace(field_value[q])) { + ++q; + } + // Here's our first scheme. + string scheme = downcase(field_value.substr(p, q - p)); + Tokens *tokens = &(schemes[scheme]); + + // Now pull off the tokens, one at a time. + p = q + 1; + while (p < field_value.length()) { + q = p; + while (q < field_value.length() && field_value[q] != '=' && + field_value[q] != ',' && !isspace(field_value[q])) { + ++q; + } + if (field_value[q] == '=') { + // This is a token. + string token = downcase(field_value.substr(p, q - p)); + string value; + p = scan_quoted_or_unquoted_string(value, field_value, q + 1); + (*tokens)[token] = value; + + // Skip trailing whitespace and extra commas. + while (p < field_value.length() && + (field_value[p] == ',' || isspace(field_value[p]))) { + ++p; + } + + } else { + // This is not a token; it must be the start of a new scheme. + scheme = downcase(field_value.substr(p, q - p)); + tokens = &(schemes[scheme]); + p = q + 1; + } + } + } +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::read_body +// Access: Private +// Description: Returns a newly-allocated istream suitable for +// reading the body of the document. If owns_source is +// true, the ownership of the _source pointer will be +// passed to the istream; otherwise, it will be +// retained. (owns_source must be true in order to read +// "identity" encoded documents.) +//////////////////////////////////////////////////////////////////// +istream *HTTPDocument:: +read_body(bool owns_source) { + if (_state != S_read_header || _source == (IBioStream *)NULL) { + return NULL; + } + + string transfer_coding = downcase(get_header_value("Transfer-Encoding")); + string content_length = get_header_value("Content-Length"); + + istream *result; + if (transfer_coding == "chunked") { + // Chunked can be used directly. + _file_size = 0; + _state = S_started_body; + _read_index++; + result = new IChunkedStream(_source, owns_source, (HTTPDocument *)this); + if (owns_source) { + _source = (IBioStream *)NULL; + } + + } else if (!content_length.empty()) { + // If we have a content length, we can use an IdentityStream. + _state = S_started_body; + _read_index++; + result = new IIdentityStream(_source, owns_source, (HTTPDocument *)this, _file_size); + if (owns_source) { + _source = (IBioStream *)NULL; + } + + } else if (owns_source) { + // If we own the source, we can return it. + _state = S_started_body; + result = _source; + _source = (IBioStream *)NULL; + + } else { + // Otherwise, we don't own the source; too bad. + result = NULL; + } + + return result; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::prepare_for_next +// Access: Private +// Description: Resets the state to prepare it for sending a new +// request to the server. This might mean closing the +// connection and opening a new one, or it might mean +// skipping past the unread body in the persistent +// connection, or it might do nothing at all if the body +// has already been completely read. +//////////////////////////////////////////////////////////////////// +bool HTTPDocument:: +prepare_for_next() { + if (get_persistent_connection() && !will_close_connection() && + _proxy == _client->get_proxy()) { + // See if we can reuse the current connection. + if (_state == S_read_header) { + // We have read the header; now skip past the body. + istream *body = read_body(false); + if (body != (istream *)NULL) { + string line; + getline(*body, line); + while (!body->fail() && !body->eof()) { + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "skip: " << line << "\n"; + } + getline(*body, line); + } + nassertr(body != _source, false); + delete body; + } + } + + if (_source == (IBioStream *)NULL) { + _source = new IBioStream(_bio, false); + } + + if (_state == S_read_body) { + // We have read the body, but there's a trailer to read. + string line; + getline(*_source, line); + if (!line.empty() && line[line.length() - 1] == '\r') { + line = line.substr(0, line.length() - 1); + } + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "skip: " << line << "\n"; + } + while (!_source->eof() && !_source->fail() && !line.empty()) { + getline(*_source, line); + if (!line.empty() && line[line.length() - 1] == '\r') { + line = line.substr(0, line.length() - 1); + } + if (downloader_cat.is_spam()) { + downloader_cat.spam() << "skip: " << line << "\n"; + } + } + _state = S_read_trailer; + } + + if (_state == S_read_trailer) { + // Great; this connection is ready to go! + return true; + } + } + + if (_bio != (BIO *)NULL && _state == S_new) { + // If we have a BIO and the _state is S_new, then we haven't done + // anything with the BIO yet, so we can still use it. + if (_source == (IBioStream *)NULL) { + _source = new IBioStream(_bio, false); + } + return true; + } + + // Either the client will close the connection after reading the + // body, or we were only partly through reading the body elsewhere; + // or possibly we don't have a connection yet at all. In any case, + // we must now get a new connection. + if (_bio != (BIO *)NULL && !_owns_bio) { + // We have a connection, but we don't own it, so we can't close + // it. Too bad. + return false; + } + + // Go ahead and close the old BIO. + free_bio(); + + _proxy = _client->get_proxy(); + _http_version = _client->get_http_version(); + _http_version_string = _client->get_http_version_string(); + _bio = _client->establish_connection(_url); + _owns_bio = true; + if (_bio != (BIO *)NULL) { + _source = new IBioStream(_bio, false); + return true; + } + return false; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::free_bio +// Access: Private +// Description: Frees the BIO and its IBioStream object, if +// allocated. This will close the connection if it is +// open. +//////////////////////////////////////////////////////////////////// +void HTTPDocument:: +free_bio() { + if (_source != (IBioStream *)NULL) { + delete _source; + _source = (IBioStream *)NULL; + } + if (_bio != (BIO *)NULL) { + if (_owns_bio) { + // TODO: We should be more careful here to manage reference + // counts so we don't free the bio out from under a BIOStreamBuf + // that's trying to read from it. + if (downloader_cat.is_debug()) { + const URLSpec &url = _proxy.empty() ? _url : _proxy; + downloader_cat.debug() + << "Dropping connection to " << url.get_server() << "\n"; + } + BIO_free_all(_bio); + } + _bio = (BIO *)NULL; + } + _read_index++; + _state = S_new; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPDocument::get_basic_authorization +// Access: Private +// Description: Looks for a username:password to satisfy the "Basic" +// scheme authorization request from the server or +// proxy. +//////////////////////////////////////////////////////////////////// +bool HTTPDocument:: +get_basic_authorization(string &authorization, const HTTPDocument::Tokens &tokens, const URLSpec &url, bool is_proxy) { + Tokens::const_iterator ti; + ti = tokens.find("realm"); + if (ti != tokens.end()) { + _realm = (*ti).second; + } + + string username; + + // Look in several places in order to find the matching username. + + // Fist, if there's a username on the URL, that always wins. + if (url.has_username()) { + username = url.get_username(); + } + + // Otherwise, start looking on the HTTPClient. + if (is_proxy) { + if (username.empty()) { + // Try the *proxy/realm. + username = _client->get_username("*proxy", _realm); + } + if (username.empty()) { + // Then, try *proxy/any realm. + username = _client->get_username("*proxy", string()); + } + } + if (username.empty()) { + // Try the specific server/realm. + username = _client->get_username(url.get_server(), _realm); + } + if (username.empty()) { + // Then, try the specific server/any realm. + username = _client->get_username(url.get_server(), string()); + } + if (username.empty()) { + // Then, try any server with this realm. + username = _client->get_username(string(), _realm); + } + if (username.empty()) { + // Then, take the general password. + username = _client->get_username(string(), string()); + } + + if (username.empty()) { + // No username:password available. + return false; + } + + authorization = "Basic " + base64_encode(username); + return true; +} #endif // HAVE_SSL diff --git a/panda/src/downloader/httpDocument.h b/panda/src/downloader/httpDocument.h index 7670366dc2..fcfb848f3a 100644 --- a/panda/src/downloader/httpDocument.h +++ b/panda/src/downloader/httpDocument.h @@ -28,20 +28,28 @@ #ifdef HAVE_SSL +#include "httpClient.h" #include "urlSpec.h" #include "virtualFile.h" #include "pmap.h" #include class IBioStream; +class HTTPClient; //////////////////////////////////////////////////////////////////// // Class : HTTPDocument // Description : A single document retrieved from an HTTP server. //////////////////////////////////////////////////////////////////// class EXPCL_PANDAEXPRESS HTTPDocument : public VirtualFile { +private: + HTTPDocument(HTTPClient *client, BIO *bio = NULL); + + bool send_request(const string &method, const URLSpec &url, + const string &body); + bool send_request(const string &header, const string &body); + public: - HTTPDocument(BIO *bio, bool owns_bio); virtual ~HTTPDocument(); virtual VirtualFileSystem *get_file_system() const; @@ -50,32 +58,90 @@ public: virtual bool is_regular_file() const; virtual istream *open_read_file() const; + bool will_close_connection() const; + PUBLISHED: INLINE bool is_valid() const; - INLINE const string &get_http_version() const; + INLINE const URLSpec &get_url() const; + INLINE HTTPClient::HTTPVersion get_http_version() const; + INLINE const string &get_http_version_string() const; INLINE int get_status_code() const; INLINE const string &get_status_string() const; + INLINE const string &get_realm() const; + INLINE const URLSpec &get_redirect() const; string get_header_value(const string &key) const; + INLINE void set_persistent_connection(bool persistent_connection); + INLINE bool get_persistent_connection() const; + INLINE size_t get_file_size() const; void write_headers(ostream &out) const; + INLINE bool get_document(const URLSpec &url, const string &body = string()); + INLINE bool get_header(const URLSpec &url); + private: - void read_headers(); - void determine_content_length(); + void make_header(string &header, const string &method, + const URLSpec &url, const string &body); + void set_url(const URLSpec &url); + void issue_request(const string &header, const string &body); + void read_http_response(); + void store_header_field(const string &field_name, const string &field_value); + bool get_authorization(string &authorization, + const string &authenticate_request, + const URLSpec &url, bool is_proxy); + static string downcase(const string &s); + static string base64_encode(const string &s); + static size_t scan_quoted_or_unquoted_string(string &result, const string &source, size_t start); + +#ifndef NDEBUG + static void show_send(const string &message); +#endif + + istream *read_body(bool owns_source); + bool prepare_for_next(); + void free_bio(); + + HTTPClient *_client; + URLSpec _proxy; + BIO *_bio; + bool _owns_bio; IBioStream *_source; + bool _persistent_connection; - string _http_version; + URLSpec _url; + string _method; + + enum State { + S_new, + S_read_header, + S_started_body, + S_read_body, + S_read_trailer + }; + State _state; + int _read_index; + + HTTPClient::HTTPVersion _http_version; + string _http_version_string; int _status_code; string _status_string; + string _realm; + URLSpec _redirect; typedef pmap Headers; Headers _headers; size_t _file_size; + typedef pmap Tokens; + typedef pmap AuthenticationSchemes; + static void parse_authentication_schemes(AuthenticationSchemes &schemes, + const string &field_value); + bool get_basic_authorization(string &authorization, const Tokens &tokens, + const URLSpec &url, bool is_proxy); public: virtual TypeHandle get_type() const { @@ -94,6 +160,8 @@ public: private: static TypeHandle _type_handle; friend class ChunkedStreamBuf; + friend class IdentityStreamBuf; + friend class HTTPClient; }; #include "httpDocument.I" diff --git a/panda/src/downloader/identityStream.I b/panda/src/downloader/identityStream.I new file mode 100644 index 0000000000..b8ab30af1d --- /dev/null +++ b/panda/src/downloader/identityStream.I @@ -0,0 +1,63 @@ +// Filename: identityStream.I +// Created by: drose (09Oct02) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://www.panda3d.org/license.txt . +// +// To contact the maintainers of this program write to +// panda3d@yahoogroups.com . +// +//////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////// +// Function: IIdentityStream::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE IIdentityStream:: +IIdentityStream() : istream(&_buf) { +} + +//////////////////////////////////////////////////////////////////// +// Function: IIdentityStream::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE IIdentityStream:: +IIdentityStream(istream *source, bool owns_source, + HTTPDocument *doc, size_t content_length) : istream(&_buf) { + open(source, owns_source, doc, content_length); +} + +//////////////////////////////////////////////////////////////////// +// Function: IIdentityStream::open +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +INLINE IIdentityStream &IIdentityStream:: +open(istream *source, bool owns_source, HTTPDocument *doc, + size_t content_length) { + clear(0); + _buf.open_read(source, owns_source, doc, content_length); + return *this; +} + +//////////////////////////////////////////////////////////////////// +// Function: IIdentityStream::close +// Access: Public +// Description: Resets the IdentityStream to empty, but does not actually +// close the source IDENTITY unless owns_source was true. +//////////////////////////////////////////////////////////////////// +INLINE IIdentityStream &IIdentityStream:: +close() { + _buf.close_read(); + return *this; +} diff --git a/panda/src/downloader/identityStream.cxx b/panda/src/downloader/identityStream.cxx new file mode 100644 index 0000000000..1598272e58 --- /dev/null +++ b/panda/src/downloader/identityStream.cxx @@ -0,0 +1,19 @@ +// Filename: identityStream.cxx +// Created by: drose (09Oct02) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://www.panda3d.org/license.txt . +// +// To contact the maintainers of this program write to +// panda3d@yahoogroups.com . +// +//////////////////////////////////////////////////////////////////// + +#include "identityStream.h" diff --git a/panda/src/downloader/identityStream.h b/panda/src/downloader/identityStream.h new file mode 100644 index 0000000000..86b8818a79 --- /dev/null +++ b/panda/src/downloader/identityStream.h @@ -0,0 +1,60 @@ +// Filename: identityStream.h +// Created by: drose (09Oct02) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://www.panda3d.org/license.txt . +// +// To contact the maintainers of this program write to +// panda3d@yahoogroups.com . +// +//////////////////////////////////////////////////////////////////// + +#ifndef IDENTITYSTREAM_H +#define IDENTITYSTREAM_H + +#include "pandabase.h" + +#include "identityStreamBuf.h" + +class HTTPDocument; + +//////////////////////////////////////////////////////////////////// +// Class : IIdentityStream +// Description : An input stream object that reads data from a source +// istream, but automatically decodes the "identity" +// transfer-coding specified by an HTTP server. +// +// In practice, this just means it reads from the sub +// stream (like a SubStreamBuf) up to but not past the +// specified content-length. (If the content-length was +// unspecified, this class cannot be used.) It also +// updates the HTTPDocument when the stream is +// completely read. +//////////////////////////////////////////////////////////////////// +// No need to export from DLL. +class IIdentityStream : public istream { +public: + INLINE IIdentityStream(); + INLINE IIdentityStream(istream *source, bool owns_source, + HTTPDocument *doc, size_t content_length); + + INLINE IIdentityStream &open(istream *source, bool owns_source, + HTTPDocument *doc, size_t content_length); + INLINE IIdentityStream &close(); + +private: + IdentityStreamBuf _buf; +}; + +#include "identityStream.I" + +#endif + + diff --git a/panda/src/downloader/identityStreamBuf.cxx b/panda/src/downloader/identityStreamBuf.cxx new file mode 100644 index 0000000000..d88ab720dc --- /dev/null +++ b/panda/src/downloader/identityStreamBuf.cxx @@ -0,0 +1,160 @@ +// Filename: identityStreamBuf.cxx +// Created by: drose (09Oct02) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://www.panda3d.org/license.txt . +// +// To contact the maintainers of this program write to +// panda3d@yahoogroups.com . +// +//////////////////////////////////////////////////////////////////// + +#include "identityStreamBuf.h" + +#ifndef HAVE_STREAMSIZE +// Some compilers (notably SGI) don't define this for us +typedef int streamsize; +#endif /* HAVE_STREAMSIZE */ + +//////////////////////////////////////////////////////////////////// +// Function: IdentityStreamBuf::Constructor +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +IdentityStreamBuf:: +IdentityStreamBuf() { + _source = (istream *)NULL; + _owns_source = false; + _bytes_remaining = 0; + +#ifdef WIN32_VC + // In spite of the claims of the MSDN Library to the contrary, + // Windows doesn't seem to provide an allocate() function, so we'll + // do it by hand. + char *buf = new char[4096]; + char *ebuf = buf + 4096; + setg(buf, ebuf, ebuf); + setp(buf, ebuf); + +#else + allocate(); + setg(base(), ebuf(), ebuf()); + setp(base(), ebuf()); +#endif +} + +//////////////////////////////////////////////////////////////////// +// Function: IdentityStreamBuf::Destructor +// Access: Public, Virtual +// Description: +//////////////////////////////////////////////////////////////////// +IdentityStreamBuf:: +~IdentityStreamBuf() { + close_read(); +} + +//////////////////////////////////////////////////////////////////// +// Function: IdentityStreamBuf::open_read +// Access: Public +// Description: If the document pointer is non-NULL, it will be +// updated with the length of the file as it is derived +// from the identity encoding. +//////////////////////////////////////////////////////////////////// +void IdentityStreamBuf:: +open_read(istream *source, bool owns_source, HTTPDocument *doc, + size_t content_length) { + _source = source; + _owns_source = owns_source; + _doc = doc; + _bytes_remaining = content_length; + + if (_doc != (HTTPDocument *)NULL) { + _read_index = doc->_read_index; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: IdentityStreamBuf::close_read +// Access: Public +// Description: +//////////////////////////////////////////////////////////////////// +void IdentityStreamBuf:: +close_read() { + if (_source != (istream *)NULL) { + if (_owns_source) { + delete _source; + _owns_source = false; + } + _source = (istream *)NULL; + } +} + +//////////////////////////////////////////////////////////////////// +// Function: IdentityStreamBuf::underflow +// Access: Protected, Virtual +// Description: Called by the system istream implementation when its +// internal buffer needs more characters. +//////////////////////////////////////////////////////////////////// +int IdentityStreamBuf:: +underflow() { + // Sometimes underflow() is called even if the buffer is not empty. + if (gptr() >= egptr()) { + size_t buffer_size = egptr() - eback(); + gbump(-(int)buffer_size); + + size_t num_bytes = buffer_size; + size_t read_count = read_chars(gptr(), buffer_size); + + if (read_count != num_bytes) { + // Oops, we didn't read what we thought we would. + if (read_count == 0) { + return EOF; + } + + // Slide what we did read to the top of the buffer. + nassertr(read_count < num_bytes, EOF); + size_t delta = num_bytes - read_count; + memmove(gptr() + delta, gptr(), read_count); + gbump(delta); + } + } + + return (unsigned char)*gptr(); +} + + +//////////////////////////////////////////////////////////////////// +// Function: IdentityStreamBuf::read_chars +// Access: Private +// Description: Gets some characters from the source stream. +//////////////////////////////////////////////////////////////////// +size_t IdentityStreamBuf:: +read_chars(char *start, size_t length) { + if (_bytes_remaining == 0) { + return 0; + } + + // Extract some of the bytes remaining in the chunk. + length = min(length, _bytes_remaining); + _source->read(start, length); + length = _source->gcount(); + _bytes_remaining -= length; + + if (_bytes_remaining == 0) { + // We're done. + if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) { + // An IdentityStreamBuf doesn't have a trailer, so we've already + // "read" it. + _doc->_state = HTTPDocument::S_read_trailer; + } + } + + return length; +} diff --git a/panda/src/downloader/identityStreamBuf.h b/panda/src/downloader/identityStreamBuf.h new file mode 100644 index 0000000000..5ff3c35e9c --- /dev/null +++ b/panda/src/downloader/identityStreamBuf.h @@ -0,0 +1,54 @@ +// Filename: identityStreamBuf.h +// Created by: drose (09Oct02) +// +//////////////////////////////////////////////////////////////////// +// +// PANDA 3D SOFTWARE +// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved +// +// All use of this software is subject to the terms of the Panda 3d +// Software license. You should have received a copy of this license +// along with this source code; you will also find a current copy of +// the license at http://www.panda3d.org/license.txt . +// +// To contact the maintainers of this program write to +// panda3d@yahoogroups.com . +// +//////////////////////////////////////////////////////////////////// + +#ifndef IDENTITYSTREAMBUF_H +#define IDENTITYSTREAMBUF_H + +#include "pandabase.h" +#include "httpDocument.h" +#include "pointerTo.h" + +//////////////////////////////////////////////////////////////////// +// Class : IdentityStreamBuf +// Description : The streambuf object that implements +// IIdentityStream. +//////////////////////////////////////////////////////////////////// +class EXPCL_PANDAEXPRESS IdentityStreamBuf : public streambuf { +public: + IdentityStreamBuf(); + virtual ~IdentityStreamBuf(); + + void open_read(istream *source, bool owns_source, HTTPDocument *doc, + size_t content_length); + void close_read(); + +protected: + virtual int underflow(void); + +private: + size_t read_chars(char *start, size_t length); + + istream *_source; + bool _owns_source; + size_t _bytes_remaining; + + PT(HTTPDocument) _doc; + int _read_index; +}; + +#endif diff --git a/panda/src/downloader/urlSpec.I b/panda/src/downloader/urlSpec.I index 299a54eed5..e33d9d1a47 100644 --- a/panda/src/downloader/urlSpec.I +++ b/panda/src/downloader/urlSpec.I @@ -47,6 +47,36 @@ operator = (const string &url) { set_url(url); } +//////////////////////////////////////////////////////////////////// +// Function: URLSpec::Operator == +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +INLINE bool URLSpec:: +operator == (const URLSpec &other) const { + return _url == other._url; +} + +//////////////////////////////////////////////////////////////////// +// Function: URLSpec::Operator != +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +INLINE bool URLSpec:: +operator != (const URLSpec &other) const { + return !operator == (other); +} + +//////////////////////////////////////////////////////////////////// +// Function: URLSpec::Operator < +// Access: Published +// Description: +//////////////////////////////////////////////////////////////////// +INLINE bool URLSpec:: +operator < (const URLSpec &other) const { + return _url < other._url; +} + //////////////////////////////////////////////////////////////////// // Function: URLSpec::has_scheme // Access: Published diff --git a/panda/src/downloader/urlSpec.h b/panda/src/downloader/urlSpec.h index d08165eede..66ea971baa 100644 --- a/panda/src/downloader/urlSpec.h +++ b/panda/src/downloader/urlSpec.h @@ -38,6 +38,10 @@ PUBLISHED: INLINE void operator = (const string &url); void operator = (const URLSpec ©); + INLINE bool operator == (const URLSpec &other) const; + INLINE bool operator != (const URLSpec &other) const; + INLINE bool operator < (const URLSpec &other) const; + INLINE bool has_scheme() const; INLINE bool has_authority() const; INLINE bool has_username() const;