From ed6187271cd5dbd4252a01f4f1bcbc4a7f9b7e36 Mon Sep 17 00:00:00 2001 From: David Rose Date: Tue, 12 Aug 2003 21:07:59 +0000 Subject: [PATCH] refine interface for determining file size of download --- panda/src/downloader/chunkedStreamBuf.cxx | 13 +++- panda/src/downloader/config_downloader.cxx | 4 ++ panda/src/downloader/config_downloader.h | 2 + panda/src/downloader/httpChannel.I | 80 ++++++++++++++++++---- panda/src/downloader/httpChannel.cxx | 79 ++++++++++++++++----- panda/src/downloader/httpChannel.h | 12 +++- panda/src/downloader/httpClient.I | 27 ++++++++ panda/src/downloader/httpClient.cxx | 35 ++-------- panda/src/downloader/httpClient.h | 7 +- 9 files changed, 191 insertions(+), 68 deletions(-) diff --git a/panda/src/downloader/chunkedStreamBuf.cxx b/panda/src/downloader/chunkedStreamBuf.cxx index c19d75bb40..b64df1ac20 100644 --- a/panda/src/downloader/chunkedStreamBuf.cxx +++ b/panda/src/downloader/chunkedStreamBuf.cxx @@ -17,6 +17,7 @@ //////////////////////////////////////////////////////////////////// #include "chunkedStreamBuf.h" +#include "config_downloader.h" #include // This module is not compiled if OpenSSL is not available. @@ -77,7 +78,8 @@ open_read(BioStreamPtr *source, HTTPChannel *doc) { if (_doc != (HTTPChannel *)NULL) { _read_index = doc->_read_index; - _doc->_file_size = 0; + _doc->_transfer_file_size = 0; + _doc->_got_transfer_file_size = true; // Read a little bit from the file to get the first chunk (and // therefore the file size, or at least the size of the first @@ -180,9 +182,16 @@ read_chars(char *start, size_t length) { return 0; } size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16); + if (downloader_cat.is_spam()) { + downloader_cat.spam() + << "Got chunk of size " << chunk_size << " bytes.\n"; + } + if (chunk_size == 0) { // Last chunk; we're done. _done = true; + _doc->_file_size = _doc->_transfer_file_size; + _doc->_got_file_size = true; if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { _doc->finished_body(true); } @@ -190,7 +199,7 @@ read_chars(char *start, size_t length) { } if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) { - _doc->_file_size += chunk_size; + _doc->_transfer_file_size += chunk_size; } _chunk_remaining = chunk_size; diff --git a/panda/src/downloader/config_downloader.cxx b/panda/src/downloader/config_downloader.cxx index 7482e6e2ac..91ffb7fddc 100644 --- a/panda/src/downloader/config_downloader.cxx +++ b/panda/src/downloader/config_downloader.cxx @@ -88,8 +88,12 @@ const string http_proxy = config_downloader.GetString("http-proxy", ""); const string http_direct_hosts = config_downloader.GetString("http-direct-hosts", ""); +const bool http_try_all_direct = +config_downloader.GetBool("http-try-all-direct", true); const string http_proxy_username = config_downloader.GetString("http-proxy-username", ""); +const bool http_proxy_tunnel = +config_downloader.GetBool("http-proxy-tunnel", false); // This is the default amount of time to wait for a TCP/IP connection // to be established, in seconds. It is presently only used for diff --git a/panda/src/downloader/config_downloader.h b/panda/src/downloader/config_downloader.h index 06ffa53ec1..3591074730 100644 --- a/panda/src/downloader/config_downloader.h +++ b/panda/src/downloader/config_downloader.h @@ -45,7 +45,9 @@ extern const bool verify_ssl; extern const string ssl_cipher_list; extern const string http_proxy; extern const string http_direct_hosts; +extern const bool http_try_all_direct; extern const string http_proxy_username; +extern const bool http_proxy_tunnel; extern const double connect_timeout; extern const double http_timeout; extern const int http_max_connect_count; diff --git a/panda/src/downloader/httpChannel.I b/panda/src/downloader/httpChannel.I index c8baf54636..537275d6f5 100644 --- a/panda/src/downloader/httpChannel.I +++ b/panda/src/downloader/httpChannel.I @@ -241,6 +241,44 @@ get_allow_proxy() const { return _allow_proxy; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::set_proxy_tunnel +// Access: Published +// Description: Normally, a proxy is itself asked for ordinary URL's, +// and the proxy decides whether to hand the client a +// cached version of the document or to contact the +// server for a fresh version. The proxy may also +// modify the headers and transfer encoding on the way. +// +// If this is set to true, then instead of asking for +// URL's from the proxy, we will ask the proxy to open a +// connection to the server (for instance, on port 80); +// if the proxy honors this request, then we contact the +// server directly through this connection to retrieve +// the document. If the proxy does not honor the +// connect request, then the retrieve operation fails. +// +// SSL connections (e.g. https), and connections through +// a Socks proxy, are always tunneled, regardless of the +// setting of this flag. +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +set_proxy_tunnel(bool proxy_tunnel) { + _proxy_tunnel = proxy_tunnel; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::get_proxy_tunnel +// Access: Published +// Description: Returns true if connections always tunnel through a +// proxy, or false (the normal case) if we allow the +// proxy to serve up documents. See set_proxy_tunnel(). +//////////////////////////////////////////////////////////////////// +INLINE bool HTTPChannel:: +get_proxy_tunnel() const { + return _proxy_tunnel; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::set_connect_timeout // Access: Published @@ -432,22 +470,36 @@ get_max_updates_per_second() const { } //////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::get_file_size +// Function: HTTPChannel::set_expected_file_size // Access: Published -// Description: Returns the size of the file, if it is known. -// Returns 0 if the file size is not known. -// -// If the file is dynamically generated, the size may -// not be available until a read has started -// (e.g. open_read_file() has been called); and even -// then it may increase as more of the file is read due -// to the nature of HTTP/1.1 requests which can change -// their minds midstream about how much data they're -// sending you. +// Description: This may be called immediately after a call to +// get_document() or some related function to specify +// the expected size of the document we are retrieving, +// if we happen to know. This is used as the return +// value to get_file_size() only in the case that the +// server does not tell us the actual file size. //////////////////////////////////////////////////////////////////// -INLINE size_t HTTPChannel:: -get_file_size() const { - return _file_size; +INLINE void HTTPChannel:: +set_expected_file_size(size_t file_size) { + _expected_file_size = file_size; + _got_expected_file_size = true; +} + + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::is_file_size_known +// Access: Published +// Description: Returns true if the size of the file we are currently +// retrieving was told us by the server and thus is +// reliably known, or false if the size reported by +// get_file_size() represents an educated guess +// (possibly as set by set_expected_file_size(), or as +// inferred from a chunked transfer encoding in +// progress). +//////////////////////////////////////////////////////////////////// +INLINE bool HTTPChannel:: +is_file_size_known() const { + return _got_file_size; } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/downloader/httpChannel.cxx b/panda/src/downloader/httpChannel.cxx index a7f3cab6ca..4adcc07990 100644 --- a/panda/src/downloader/httpChannel.cxx +++ b/panda/src/downloader/httpChannel.cxx @@ -44,6 +44,7 @@ HTTPChannel(HTTPClient *client) : _proxy_next_index = 0; _persistent_connection = false; _allow_proxy = true; + _proxy_tunnel = http_proxy_tunnel; _connect_timeout = connect_timeout; _http_timeout = http_timeout; _blocking_connect = false; @@ -55,13 +56,18 @@ HTTPChannel(HTTPClient *client) : _nonblocking = false; _want_ssl = false; _proxy_serves_document = false; - _proxy_tunnel = false; + _proxy_tunnel_now = false; _first_byte_requested = 0; _last_byte_requested = 0; _first_byte_delivered = 0; _last_byte_delivered = 0; _read_index = 0; + _expected_file_size = 0; _file_size = 0; + _transfer_file_size = 0; + _got_expected_file_size = false; + _got_file_size = false; + _got_transfer_file_size = false; _bytes_downloaded = 0; _bytes_requested = 0; _status_code = 0; @@ -179,6 +185,35 @@ get_header_value(const string &key) const { return string(); } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::get_file_size +// Access: Published +// Description: Returns the size of the file, if it is known. +// Returns the value set by set_expected_file_size() if +// the file size is not known, or 0 if this value was +// not set. +// +// If the file is dynamically generated, the size may +// not be available until a read has started +// (e.g. open_read_file() has been called); and even +// then it may increase as more of the file is read due +// to the nature of HTTP/1.1 requests which can change +// their minds midstream about how much data they're +// sending you. +//////////////////////////////////////////////////////////////////// +size_t HTTPChannel:: +get_file_size() const { + if (_got_file_size) { + return _file_size; + } else if (_got_transfer_file_size) { + return _transfer_file_size; + } else if (_got_expected_file_size) { + return _expected_file_size; + } else { + return 0; + } +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::write_headers // Access: Published @@ -421,17 +456,15 @@ read_body() { } string transfer_coding = downcase(get_header_value("Transfer-Encoding")); - string content_length = get_header_value("Content-Length"); ISocketStream *result; if (transfer_coding == "chunked") { // "chunked" transfer encoding. This means we will have to decode // the length of the file as we read it in chunks. The // IChunkedStream does this. - _file_size = 0; _state = S_reading_body; _read_index++; - result = new IChunkedStream(_source, (HTTPChannel *)this); + result = new IChunkedStream(_source, this); } else { // If the transfer encoding is anything else, assume "identity". @@ -440,8 +473,7 @@ read_body() { // specified), or till end of file otherwise. _state = S_reading_body; _read_index++; - result = new IIdentityStream(_source, (HTTPChannel *)this, - !content_length.empty(), _file_size); + result = new IIdentityStream(_source, this, _got_file_size, _file_size); } return result; @@ -709,7 +741,7 @@ run_connecting() { << _bio->get_port() << "\n"; } - if (_proxy_tunnel) { + if (_proxy_tunnel_now) { if (_proxy.get_scheme() == "socks") { _state = S_socks_proxy_greet; } else { @@ -833,7 +865,8 @@ run_http_proxy_request_sent() { _current_field_name = string(); _current_field_value = string(); _headers.clear(); - _file_size = 0; + _got_file_size = false; + _got_transfer_file_size = false; return false; } @@ -1355,7 +1388,8 @@ run_request_sent() { _current_field_name = string(); _current_field_value = string(); _headers.clear(); - _file_size = 0; + _got_file_size = false; + _got_transfer_file_size = false; return false; } @@ -1451,15 +1485,20 @@ run_reading_header() { return false; } - _file_size = 0; + _got_expected_file_size = false; + _got_file_size = false; + _got_transfer_file_size = false; + string content_length = get_header_value("Content-Length"); if (!content_length.empty()) { _file_size = atoi(content_length.c_str()); + _got_file_size = true; } else if (get_status_code() == 206) { // Well, we didn't get a content-length from the server, but we // can infer the number of bytes based on the range we're given. _file_size = _last_byte_delivered - _first_byte_delivered + 1; + _got_file_size = true; } _redirect = get_header_value("Location"); @@ -1608,15 +1647,15 @@ run_begin_body() { // We have already "read" the nonexistent body. _state = S_read_trailer; - } else if (_file_size > 8192) { + } else if (get_file_size() > 8192) { // If we know the size of the body we are about to skip and it's // too large (and here we arbitrarily say 8KB is too large), then // don't bother skipping it--just drop the connection and get a // new one. if (downloader_cat.is_debug()) { downloader_cat.debug() - << "Dropping connection rather than skipping past " << _file_size - << " bytes.\n"; + << "Dropping connection rather than skipping past " + << get_file_size() << " bytes.\n"; } reset_to_new(); @@ -1920,26 +1959,28 @@ begin_request(HTTPEnum::Method method, const DocumentSpec &url, //////////////////////////////////////////////////////////////////// void HTTPChannel:: reconsider_proxy() { - _proxy_tunnel = false; + _proxy_tunnel_now = false; _proxy_serves_document = false; if (!_proxy.empty()) { - // If we're opening an SSL connection, or the user has explicitly + // If the user insists we always tunnel through a proxy, or if + // we're opening an SSL connection, or the user has explicitly // asked for a direct connection of some kind, or if we have a // SOCKS-style proxy; each of these demands a tunnel through the // proxy to speak directly to the http server. - _proxy_tunnel = - (_want_ssl || _method == HTTPEnum::M_connect || _proxy.get_scheme() == "socks"); + _proxy_tunnel_now = + (get_proxy_tunnel() || _want_ssl || + _method == HTTPEnum::M_connect || _proxy.get_scheme() == "socks"); // Otherwise (but we still have a proxy), then we ask the proxy to // hand us the document. - _proxy_serves_document = !_proxy_tunnel; + _proxy_serves_document = !_proxy_tunnel_now; } make_header(); make_request_text(); - if (_proxy_tunnel) { + if (_proxy_tunnel_now) { // Maybe we need to tunnel through the proxy to connect to the // server directly. ostringstream request; diff --git a/panda/src/downloader/httpChannel.h b/panda/src/downloader/httpChannel.h index ec6f69cde9..1d7c68a44f 100644 --- a/panda/src/downloader/httpChannel.h +++ b/panda/src/downloader/httpChannel.h @@ -93,6 +93,8 @@ PUBLISHED: INLINE void set_allow_proxy(bool allow_proxy); INLINE bool get_allow_proxy() const; + INLINE void set_proxy_tunnel(bool proxy_tunnel); + INLINE bool get_proxy_tunnel() const; INLINE void set_connect_timeout(double timeout_seconds); INLINE double get_connect_timeout() const; @@ -111,7 +113,9 @@ PUBLISHED: INLINE void set_max_updates_per_second(double max_updates_per_second); INLINE double get_max_updates_per_second() const; + INLINE void set_expected_file_size(size_t file_size); INLINE size_t get_file_size() const; + INLINE bool is_file_size_known() const; void write_headers(ostream &out) const; @@ -255,6 +259,7 @@ private: PT(BioStreamPtr) _source; bool _persistent_connection; bool _allow_proxy; + bool _proxy_tunnel; double _connect_timeout; double _http_timeout; bool _blocking_connect; @@ -274,7 +279,7 @@ private: string _body; bool _want_ssl; bool _proxy_serves_document; - bool _proxy_tunnel; + bool _proxy_tunnel_now; bool _server_response_has_no_body; size_t _first_byte_requested; size_t _last_byte_requested; @@ -323,9 +328,14 @@ private: typedef pmap Headers; Headers _headers; + size_t _expected_file_size; size_t _file_size; + size_t _transfer_file_size; size_t _bytes_downloaded; size_t _bytes_requested; + bool _got_expected_file_size; + bool _got_file_size; + bool _got_transfer_file_size; // These members are used to maintain the current state while // communicating with the server. We need to store everything in diff --git a/panda/src/downloader/httpClient.I b/panda/src/downloader/httpClient.I index b9387c81bd..7a57addd53 100644 --- a/panda/src/downloader/httpClient.I +++ b/panda/src/downloader/httpClient.I @@ -17,6 +17,33 @@ //////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::set_try_all_direct +// Access: Published +// Description: If this is set true, then after a connection attempt +// through a proxy fails, we always try a direct +// connection, regardless of whether the host is listed +// on the direct_host_spec list. If this is false, a +// direct attempt is not made when we have a proxy in +// effect, even if the proxy fails. +//////////////////////////////////////////////////////////////////// +INLINE void HTTPClient:: +set_try_all_direct(bool try_all_direct) { + _try_all_direct = try_all_direct; +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPClient::get_try_all_direct +// Access: Published +// Description: Returns whether a failed connection through a proxy +// will be followed up by a direct connection attempt, +// false otherwise. +//////////////////////////////////////////////////////////////////// +INLINE bool HTTPClient:: +get_try_all_direct() const { + return _try_all_direct; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPClient::set_http_version // Access: Published diff --git a/panda/src/downloader/httpClient.cxx b/panda/src/downloader/httpClient.cxx index dc1821bec8..971039d602 100644 --- a/panda/src/downloader/httpClient.cxx +++ b/panda/src/downloader/httpClient.cxx @@ -102,6 +102,7 @@ HTTPClient() { set_proxy_spec(http_proxy); set_direct_host_spec(http_direct_hosts); + _try_all_direct = http_try_all_direct; if (!http_proxy_username.empty()) { set_username("*proxy", "", http_proxy_username); @@ -155,6 +156,7 @@ void HTTPClient:: operator = (const HTTPClient ©) { _proxies_by_scheme = copy._proxies_by_scheme; _direct_hosts = copy._direct_hosts; + _try_all_direct = copy._try_all_direct; _http_version = copy._http_version; _verify_ssl = copy._verify_ssl; _usernames = copy._usernames; @@ -190,33 +192,6 @@ HTTPClient:: clear_expected_servers(); } -//////////////////////////////////////////////////////////////////// -// Function: HTTPClient::set_proxy -// Access: Published -// Description: Specifies the proxy URL to handle all http and -// https requests. Deprecated. -//////////////////////////////////////////////////////////////////// -void HTTPClient:: -set_proxy(const URLSpec &proxy) { - set_proxy_spec(proxy.get_url()); -} - -//////////////////////////////////////////////////////////////////// -// Function: HTTPClient::get_proxy -// Access: Published -// Description: Returns the proxy URL to handle all http and -// https requests. Deprecated. -//////////////////////////////////////////////////////////////////// -URLSpec HTTPClient:: -get_proxy() const { - pvector proxies; - get_proxies_for_url(URLSpec("http://"), proxies); - if (!proxies.empty()) { - return proxies[0]; - } - return URLSpec(); -} - //////////////////////////////////////////////////////////////////// // Function: HTTPClient::set_proxy_spec // Access: Published @@ -524,8 +499,10 @@ get_proxies_for_url(const URLSpec &url, pvector &proxies) const { } } - // We always try a direct connection if all else fails. - temp_list.push_back(URLSpec()); + if (_try_all_direct) { + // We may try a direct connection if all else fails. + temp_list.push_back(URLSpec()); + } // Finally, as a very last resort, fall back to the HTTP proxy. if (!got_any) { diff --git a/panda/src/downloader/httpClient.h b/panda/src/downloader/httpClient.h index 2ca9103efa..c1321079a4 100644 --- a/panda/src/downloader/httpClient.h +++ b/panda/src/downloader/httpClient.h @@ -62,15 +62,15 @@ PUBLISHED: void operator = (const HTTPClient ©); ~HTTPClient(); - void set_proxy(const URLSpec &proxy); - URLSpec get_proxy() const; - void set_proxy_spec(const string &proxy_spec); string get_proxy_spec() const; void set_direct_host_spec(const string &direct_host_spec); string get_direct_host_spec() const; + INLINE void set_try_all_direct(bool try_all_direct); + INLINE bool get_try_all_direct() const; + void clear_proxy(); void add_proxy(const string &scheme, const URLSpec &proxy); void clear_direct_host(); @@ -141,6 +141,7 @@ private: ProxiesByScheme _proxies_by_scheme; typedef pvector DirectHosts; DirectHosts _direct_hosts; + bool _try_all_direct; HTTPEnum::HTTPVersion _http_version; VerifySSL _verify_ssl;