RomanNum3ral
/
open_toontown_panda3d
mirror of https://github.com/open-toontown/panda3d.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

robustify http handling

This commit is contained in:
David Rose 2002-10-09 23:45:46 +00:00
parent 8931a17f0a
commit 893078a291
18 changed files with 1609 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
panda/src/downloader/Sources.pp
View File

@ -18,6 +18,7 @@
extractor.h \
httpClient.I httpClient.h \
httpDocument.I httpDocument.h \
identityStream.I identityStream.h identityStreamBuf.h \
multiplexStream.I multiplexStream.h \
multiplexStreamBuf.I multiplexStreamBuf.h \
urlSpec.I urlSpec.h \
@ -33,6 +34,7 @@
extractor.cxx \
httpClient.cxx \
httpDocument.cxx \
identityStream.cxx identityStreamBuf.cxx \
multiplexStream.cxx multiplexStreamBuf.cxx \
urlSpec.cxx \
$[if $[HAVE_NET], downloadDb.cxx downloader.cxx] \
@ -49,6 +51,7 @@
extractor.h \
httpClient.I httpClient.h \
httpDocument.I httpDocument.h \
identityStream.I identityStream.h identityStreamBuf.h \
multiplexStream.I multiplexStream.h \
multiplexStreamBuf.I multiplexStreamBuf.I \
patcher.h patcher.I \

8
panda/src/downloader/chunkedStream.h
View File

@ -33,14 +33,15 @@ class HTTPDocument;
//
// Seeking is not supported.
////////////////////////////////////////////////////////////////////
class EXPCL_PANDAEXPRESS IChunkedStream : public istream {
// No need to export from DLL.
class IChunkedStream : public istream {
public:
INLINE IChunkedStream();
INLINE IChunkedStream(istream *source, bool owns_source,
HTTPDocument *doc = NULL);
HTTPDocument *doc);
INLINE IChunkedStream &open(istream *source, bool owns_source,
HTTPDocument *doc = NULL);
HTTPDocument *doc);
INLINE IChunkedStream &close();
private:
@ -52,3 +53,4 @@ private:
#endif

14
panda/src/downloader/chunkedStreamBuf.cxx
View File

@ -77,6 +77,7 @@ open_read(istream *source, bool owns_source, HTTPDocument *doc) {
_doc = doc;
if (_doc != (HTTPDocument *)NULL) {
_read_index = doc->_read_index;
_doc->_file_size = 0;
// Read a little bit from the file to get the first chunk (and
@ -162,17 +163,20 @@ read_chars(char *start, size_t length) {
if (!line.empty() && line[line.length() - 1] == '\r') {
line = line.substr(0, line.length() - 1);
}
int chunk_size = strtol(line.c_str(), NULL, 16);
if (chunk_size <= 0) {
size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16);
if (chunk_size == 0) {
// Last chunk; we're done.
_done = true;
if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) {
_doc->_state = HTTPDocument::S_read_body;
}
return 0;
}
if (_doc != (HTTPDocument *)NULL) {
_doc->_file_size += (size_t)chunk_size;
if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) {
_doc->_file_size += chunk_size;
}
_chunk_remaining = (size_t)chunk_size;
_chunk_remaining = chunk_size;
return read_chars(start, length);
}

4
panda/src/downloader/chunkedStreamBuf.h
View File

@ -28,7 +28,8 @@
// Description : The streambuf object that implements
// IChunkedStream.
////////////////////////////////////////////////////////////////////
class EXPCL_PANDAEXPRESS ChunkedStreamBuf : public streambuf {
// No need to export from DLL.
class ChunkedStreamBuf : public streambuf {
public:
ChunkedStreamBuf();
virtual ~ChunkedStreamBuf();
@ -48,6 +49,7 @@ private:
bool _done;
PT(HTTPDocument) _doc;
int _read_index;
};
#endif

2
panda/src/downloader/downloader_composite1.cxx
View File

@ -7,6 +7,8 @@
#include "extractor.cxx"
#include "httpClient.cxx"
#include "httpDocument.cxx"
#include "identityStream.cxx"
#include "identityStreamBuf.cxx"
#include "multiplexStream.cxx"
#include "multiplexStreamBuf.cxx"
#include "urlSpec.cxx"

34
panda/src/downloader/httpClient.I
View File

@ -92,37 +92,3 @@ INLINE bool HTTPClient::
get_verify_ssl() const {
return _verify_ssl;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_document
// Access: Published
// Description: Opens the named document for reading, or if body is
// nonempty, posts data for a particular URL and
// retrieves the response. Returns a new HTTPDocument
// object whether the document is successfully read or
// not; you can test is_valid() and get_return_code() to
// determine whether the document was retrieved.
////////////////////////////////////////////////////////////////////
INLINE PT(HTTPDocument) HTTPClient::
get_document(const URLSpec &url, const string &body) {
const char *method = "GET";
if (!body.empty()) {
method = "POST";
}
return make_request(method, url, body);
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_header
// Access: Published
// Description: Like get_document(), except only the header
// associated with the file is retrieved. This may be
// used to test for existence of the file; it might also
// return the size of the file (if the server gives us
// this information).
////////////////////////////////////////////////////////////////////
INLINE PT(HTTPDocument) HTTPClient::
get_header(const URLSpec &url) {
return make_request("HEAD", url, string());
}

343
panda/src/downloader/httpClient.cxx
View File

@ -17,6 +17,7 @@
////////////////////////////////////////////////////////////////////
#include "httpClient.h"
#include "httpDocument.h"
#include "config_downloader.h"
#include "filename.h"
#include "config_express.h"
@ -106,6 +107,48 @@ HTTPClient::
clear_expected_servers();
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::set_username
// Access: Published
// Description: Specifies the username:password string corresponding
// to a particular server and/or realm, when demanded by
// the server. Either or both of the server or realm
// may be empty; if so, they match anything. Also, the
// server may be set to the special string "*proxy",
// which will match any proxy server.
//
// If the username is set to the empty string, this
// clears the password for the particular server/realm
// pair.
////////////////////////////////////////////////////////////////////
void HTTPClient::
set_username(const string &server, const string &realm, const string &username) {
string key = server + ":" + realm;
if (username.empty()) {
_usernames.erase(key);
} else {
_usernames[key] = username;
}
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_username
// Access: Published
// Description: Returns the username:password string set for this
// server/realm pair, or empty string if nothing has
// been set. See set_username().
////////////////////////////////////////////////////////////////////
string HTTPClient::
get_username(const string &server, const string &realm) const {
string key = server + ":" + realm;
Usernames::const_iterator ui;
ui = _usernames.find(key);
if (ui != _usernames.end()) {
return (*ui).second;
}
return string();
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_http_version_string
// Access: Published
@ -120,11 +163,34 @@ get_http_version_string() const {
case HV_11:
return "HTTP/1.1";
case HV_other:
// Report the best we can do.
return "HTTP/1.1";
}
return "unknown";
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::parse_http_version_string
// Access: Published
// Description: Matches the string representing a particular HTTP
// version against any of the known versions and returns
// the appropriate enumerated value, or HV_other if the
// version is unknown.
////////////////////////////////////////////////////////////////////
HTTPClient::HTTPVersion HTTPClient::
parse_http_version_string(const string &version) {
if (version == "HTTP/1.0") {
return HV_10;
} else if (version == "HTTP/1.1") {
return HV_11;
} else {
return HV_other;
}
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::load_certificates
// Access: Published
@ -209,6 +275,55 @@ clear_expected_servers() {
_expected_servers.clear();
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_documents
// Access: Published
// Description: Returns a new HTTPDocument object that may be used
// for reading multiple documents using the same
// connection, for greater network efficiency than
// calling HTTPClient::get_document() repeatedly (and
// thus forcing a new connection for each document).
////////////////////////////////////////////////////////////////////
PT(HTTPDocument) HTTPClient::
get_documents() {
PT(HTTPDocument) doc = new HTTPDocument(this);
doc->set_persistent_connection(true);
return doc;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_document
// Access: Published
// Description: Opens the named document for reading, or if body is
// nonempty, posts data for a particular URL and
// retrieves the response. Returns a new HTTPDocument
// object whether the document is successfully read or
// not; you can test is_valid() and get_return_code() to
// determine whether the document was retrieved.
////////////////////////////////////////////////////////////////////
PT(HTTPDocument) HTTPClient::
get_document(const URLSpec &url, const string &body) {
PT(HTTPDocument) doc = new HTTPDocument(this);
doc->get_document(url, body);
return doc;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_header
// Access: Published
// Description: Like get_document(), except only the header
// associated with the file is retrieved. This may be
// used to test for existence of the file; it might also
// return the size of the file (if the server gives us
// this information).
////////////////////////////////////////////////////////////////////
PT(HTTPDocument) HTTPClient::
get_header(const URLSpec &url) {
PT(HTTPDocument) doc = new HTTPDocument(this);
doc->get_header(url);
return doc;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::make_ctx
@ -376,73 +491,44 @@ load_verify_locations(SSL_CTX *ctx, const Filename &ca_file) {
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::make_request
// Function: HTTPClient::establish_connection
// Access: Private
// Description: Chooses a suitable mechanism to handle this request,
// based on whether it is an http or https request, and
// according to whether we have a proxy in place.
// Issues the request and returns an HTTPDocument that
// represents the results.
// Description: Establishes a connection to the server, using the
// appropriate means. Returns the newly allocated BIO
// representing the connection, or NULL if a connection
// cannot be established.
////////////////////////////////////////////////////////////////////
PT(HTTPDocument) HTTPClient::
make_request(const string &method, const URLSpec &url, const string &body) {
BIO *HTTPClient::
establish_connection(const URLSpec &url) {
BIO *bio;
if (_proxy.empty()) {
if (url.get_scheme() == "https") {
bio = get_https(method, url, body);
bio = establish_https(url);
} else {
bio = get_http(method, url, body);
bio = establish_http(url);
}
} else {
if (url.get_scheme() == "https") {
bio = get_https_proxy(method, url, body);
bio = establish_https_proxy(url);
} else {
bio = get_http_proxy(method, url, body);
bio = establish_http_proxy(url);
}
}
return new HTTPDocument(bio, true);
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_https
// Access: Private
// Description: Opens the indicated URL directly as an ordinary http
// document.
////////////////////////////////////////////////////////////////////
BIO *HTTPClient::
get_http(const string &method, const URLSpec &url, const string &body) {
ostringstream server;
server << url.get_server() << ":" << url.get_port();
string server_str = server.str();
BIO *bio = BIO_new_connect((char *)server_str.c_str());
if (downloader_cat.is_debug()) {
downloader_cat.debug() << "connecting to " << server_str << "\n";
}
if (BIO_do_connect(bio) <= 0) {
downloader_cat.info()
<< "Could not contact server " << server_str << "\n";
#ifdef REPORT_SSL_ERRORS
ERR_print_errors_fp(stderr);
#endif
return NULL;
}
send_request(bio, method, url.get_path(), url.get_server(), body);
return bio;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_https
// Function: HTTPClient::establish_http
// Access: Private
// Description: Opens the indicated URL directly as an https
// document.
// Description: Establishes a connection to the server directly,
// without using a proxy. Returns the newly allocated
// BIO representing the connection, or NULL if a
// connection cannot be established.
////////////////////////////////////////////////////////////////////
BIO *HTTPClient::
get_https(const string &method, const URLSpec &url, const string &body) {
establish_http(const URLSpec &url) {
ostringstream server;
server << url.get_server() << ":" << url.get_port();
string server_str = server.str();
@ -461,22 +547,50 @@ get_https(const string &method, const URLSpec &url, const string &body) {
return NULL;
}
BIO *sbio = make_https_connection(bio, url);
if (sbio != (BIO *)NULL) {
send_request(sbio, method, url.get_path(), url.get_server(), body);
}
return sbio;
return bio;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_http_proxy
// Function: HTTPClient::establish_https
// Access: Private
// Description: Opens the indicated URL via the proxy as an ordinary
// http document.
// Description: Establishes a connection to the secure server
// directly, without using a proxy. Returns the newly
// allocated BIO representing the connection, or NULL if
// a connection cannot be established.
////////////////////////////////////////////////////////////////////
BIO *HTTPClient::
get_http_proxy(const string &method, const URLSpec &url, const string &body) {
establish_https(const URLSpec &url) {
ostringstream server;
server << url.get_server() << ":" << url.get_port();
string server_str = server.str();
BIO *bio = BIO_new_connect((char *)server_str.c_str());
if (downloader_cat.is_debug()) {
downloader_cat.debug() << "connecting to " << server_str << "\n";
}
if (BIO_do_connect(bio) <= 0) {
downloader_cat.info()
<< "Could not contact server " << server_str << "\n";
#ifdef REPORT_SSL_ERRORS
ERR_print_errors_fp(stderr);
#endif
return NULL;
}
return make_https_connection(bio, url);
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::establish_http_proxy
// Access: Private
// Description: Establishes a connection to the server through a
// proxy. Returns the newly allocated BIO representing
// the connection, or NULL if a connection cannot be
// established.
////////////////////////////////////////////////////////////////////
BIO *HTTPClient::
establish_http_proxy(const URLSpec &url) {
ostringstream proxy_server;
proxy_server << _proxy.get_server() << ":" << _proxy.get_port();
string proxy_server_str = proxy_server.str();
@ -496,18 +610,19 @@ get_http_proxy(const string &method, const URLSpec &url, const string &body) {
return NULL;
}
send_request(bio, method, url, url.get_server(), body);
return bio;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::get_https_proxy
// Function: HTTPClient::establish_https_proxy
// Access: Private
// Description: Opens the indicated URL via the proxy as an https
// document.
// Description: Establishes a connection to the secure server through
// a proxy. Returns the newly allocated BIO
// representing the connection, or NULL if a connection
// cannot be established.
////////////////////////////////////////////////////////////////////
BIO *HTTPClient::
get_https_proxy(const string &method, const URLSpec &url, const string &body) {
establish_https_proxy(const URLSpec &url) {
// First, ask the proxy to open a connection for us.
ostringstream proxy_server;
proxy_server << _proxy.get_server() << ":" << _proxy.get_port();
@ -528,35 +643,21 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) {
return NULL;
}
{
ostringstream request;
request
<< "CONNECT " << url.get_server() << ":" << url.get_port()
<< " " << get_http_version_string() << "\r\n"
<< "\r\n";
string request_str = request.str();
ostringstream request;
request
<< "CONNECT " << url.get_server() << ":" << url.get_port()
<< " " << get_http_version_string() << "\r\n";
string connect_header = request.str();
#ifndef NDEBUG
if (downloader_cat.is_debug()) {
show_send(request_str);
}
#endif
BIO_puts(bio, request_str.c_str());
}
// Create a temporary HTTPDocument to read the response from the
// proxy.
// Create a temporary HTTPDocument to issue the request and read the
// response from the proxy.
{
PT(HTTPDocument) doc = new HTTPDocument(bio, false);
if (!doc->is_valid()) {
PT(HTTPDocument) doc = new HTTPDocument(this, bio);
if (!doc->send_request(connect_header, string())) {
downloader_cat.info()
<< "proxy would not open connection to " << url.get_authority()
<< ": " << doc->get_status_code() << " "
<< doc->get_status_string() << "\n";
if (downloader_cat.is_debug()) {
doc->write_headers(downloader_cat.debug(false));
}
if (!get_verify_ssl()) {
// If the proxy refused to open a raw connection for us, see
@ -567,7 +668,7 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) {
// us.)
if ((doc->get_status_code() / 100) == 4) {
BIO_free_all(bio);
return get_http_proxy(method, url, body);
return establish_http_proxy(url);
}
}
return NULL;
@ -581,12 +682,7 @@ get_https_proxy(const string &method, const URLSpec &url, const string &body) {
// Ok, we now have a connection to our actual server, so start
// speaking SSL and then ask for the document we really want.
BIO *sbio = make_https_connection(bio, url);
if (sbio != (BIO *)NULL) {
send_request(sbio, method, url.get_path(), url.get_server(), body);
}
return sbio;
return make_https_connection(bio, url);
}
////////////////////////////////////////////////////////////////////
@ -850,46 +946,6 @@ certificate signing
*/
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::send_request
// Access: Private
// Description: Sends the appropriate GET or POST (or whatever)
// request to the server on the indicated connection.
////////////////////////////////////////////////////////////////////
void HTTPClient::
send_request(BIO *bio, const string &method,
const string &path, const string &server,
const string &body) const {
ostringstream request;
request
<< method << " " << path << " " << get_http_version_string() << "\r\n";
if (_http_version > HV_10) {
request
<< "Host: " << server << "\r\n"
<< "Connection: close\r\n";
}
if (!body.empty()) {
request
<< "Content-type: application/x-www-form-urlencoded\r\n"
<< "Content-Length: " << body.length() << "\r\n";
}
request
<< "\r\n"
<< body;
string request_str = request.str();
#ifndef NDEBUG
if (downloader_cat.is_debug()) {
show_send(request_str);
}
#endif
BIO_puts(bio, request_str.c_str());
}
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::verify_server
// Access: Private
@ -1089,31 +1145,6 @@ x509_name_subset(X509_NAME *name_a, X509_NAME *name_b) {
return true;
}
#ifndef NDEBUG
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::show_send
// Access: Private, Static
// Description: Writes the outgoing message, one line at a time, to
// the debugging log.
////////////////////////////////////////////////////////////////////
void HTTPClient::
show_send(const string &message) {
size_t start = 0;
size_t newline = message.find('\n', start);
while (newline != string::npos) {
downloader_cat.debug()
<< "send: " << message.substr(start, newline - start + 1);
start = newline + 1;
newline = message.find('\n', start);
}
if (start < message.length()) {
downloader_cat.debug()
<< "send: " << message.substr(start) << " (no newline)\n";
}
}
#endif // NDEBUG
#if defined(SSL_097) && !defined(NDEBUG)
////////////////////////////////////////////////////////////////////
// Function: HTTPClient::ssl_msg_callback

39
panda/src/downloader/httpClient.h
View File

@ -29,17 +29,17 @@
#ifdef HAVE_SSL
#include "urlSpec.h"
#include "httpDocument.h"
#include "pointerTo.h"
#include <openssl/ssl.h>
// Windows may define this macro inappropriately.
#ifdef WIN32
#ifdef X509_NAME
#undef X509_NAME
#endif
class Filename;
class HTTPDocument;
////////////////////////////////////////////////////////////////////
// Class : HTTPClient
@ -60,14 +60,19 @@ PUBLISHED:
INLINE void set_proxy(const URLSpec &proxy);
INLINE const URLSpec &get_proxy() const;
void set_username(const string &server, const string &realm, const string &username);
string get_username(const string &server, const string &realm) const;
enum HTTPVersion {
HV_10, // HTTP 1.0
HV_11, // HTTP 1.1
HV_other,
};
INLINE void set_http_version(HTTPVersion version);
INLINE HTTPVersion get_http_version() const;
string get_http_version_string() const;
static HTTPVersion parse_http_version_string(const string &version);
bool load_certificates(const Filename &filename);
@ -77,37 +82,29 @@ PUBLISHED:
bool add_expected_server(const string &server_attributes);
void clear_expected_servers();
INLINE PT(HTTPDocument) get_document(const URLSpec &url,
const string &body = string());
INLINE PT(HTTPDocument) get_header(const URLSpec &url);
PT(HTTPDocument) get_documents();
PT(HTTPDocument) get_document(const URLSpec &url,
const string &body = string());
PT(HTTPDocument) get_header(const URLSpec &url);
private:
void make_ctx();
static void initialize_ssl();
static int load_verify_locations(SSL_CTX *ctx, const Filename &ca_file);
PT(HTTPDocument) make_request(const string &method, const URLSpec &url,
const string &body);
BIO *get_http(const string &method, const URLSpec &url, const string &body);
BIO *get_https(const string &method, const URLSpec &url, const string &body);
BIO *get_http_proxy(const string &method, const URLSpec &url, const string &body);
BIO *get_https_proxy(const string &method, const URLSpec &url, const string &body);
BIO *establish_connection(const URLSpec &url);
BIO *establish_http(const URLSpec &url);
BIO *establish_https(const URLSpec &url);
BIO *establish_http_proxy(const URLSpec &url);
BIO *establish_https_proxy(const URLSpec &url);
BIO *make_https_connection(BIO *bio, const URLSpec &url) const;
void send_request(BIO *bio, const string &method,
const string &path, const string &server,
const string &body) const;
bool verify_server(X509_NAME *subject) const;
static X509_NAME *parse_x509_name(const string &source);
static string get_x509_name_component(X509_NAME *name, int nid);
static bool x509_name_subset(X509_NAME *name_a, X509_NAME *name_b);
#ifndef NDEBUG
static void show_send(const string &message);
#endif
#if defined(SSL_097) && !defined(NDEBUG)
static void ssl_msg_callback(int write_p, int version, int content_type,
const void *buf, size_t len, SSL *ssl,
@ -118,6 +115,9 @@ private:
HTTPVersion _http_version;
bool _verify_ssl;
typedef pmap<string, string> Usernames;
Usernames _usernames;
// List of allowable SSL servers to connect to. If the list is
// empty, any server is acceptable.
typedef pvector<X509_NAME *> ExpectedServers;
@ -127,6 +127,7 @@ private:
static bool _ssl_initialized;
static X509_STORE *_x509_store;
friend class HTTPDocument;
};
#include "httpClient.I"

133
panda/src/downloader/httpDocument.I
View File

@ -29,15 +29,39 @@ is_valid() const {
(_status_code / 100) == 2);
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_url
// Access: Published
// Description: Returns the URL that was used to retrieve this
// document: whatever URL was last passed to
// get_document() or get_header().
////////////////////////////////////////////////////////////////////
INLINE const URLSpec &HTTPDocument::
get_url() const {
return _url;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_http_version
// Access: Published
// Description: Returns the HTTP version number returned by the
// server, as one of the HTTPClient enumerated types,
// e.g. HTTPClient::HV_11.
////////////////////////////////////////////////////////////////////
INLINE HTTPClient::HTTPVersion HTTPDocument::
get_http_version() const {
return _http_version;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_http_version_string
// Access: Published
// Description: Returns the HTTP version number returned by the
// server, formatted as a string, e.g. "HTTP/1.1".
////////////////////////////////////////////////////////////////////
INLINE const string &HTTPDocument::
get_http_version() const {
return _http_version;
get_http_version_string() const {
return _http_version_string;
}
////////////////////////////////////////////////////////////////////
@ -65,6 +89,80 @@ get_status_string() const {
return _status_string;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_realm
// Access: Published
// Description: If the document failed to connect because of a 401
// (Authorization required) or 407 (Proxy authorization
// required) error, this method should return the
// "realm" returned by the server in which the requested
// document must be authenticated. This string may be
// presented to the user to request an associated
// username and password (which then should be stored in
// HTTPClient::set_username()).
//
// If the document was retrieved successfully, or failed
// to connect for some other reason, this string will be
// empty. If it is empty even in the presence of a 401
// or 407 error, then the client code and the server do
// not have a common scheme for exchanging passwords.
////////////////////////////////////////////////////////////////////
INLINE const string &HTTPDocument::
get_realm() const {
return _realm;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_redirect
// Access: Published
// Description: If the document failed with a redirect code (300
// series), this will generally contain the new URL the
// server wants us to try. In many cases, the client
// will automatically follow redirects; if these are
// succesful the client will return a successful code
// and get_redirect() will return empty, but get_url()
// will return the new, redirected URL.
////////////////////////////////////////////////////////////////////
INLINE const URLSpec &HTTPDocument::
get_redirect() const {
return _redirect;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::set_persistent_connection
// Access: Published
// Description: Indicates whether the HTTPDocument should try to keep
// the connection to the server open and reuse that
// connection for multiple documents, or whether it
// should close the connection and open a new one for
// each request. Set this true to keep the connections
// around when possible, false to recycle them.
//
// It makes most sense to set this false when the
// HTTPDocument will be used only once to retrieve a
// single document, true when you will be using the
// HTTPDocument interface to retrieve multiple
// documents.
////////////////////////////////////////////////////////////////////
INLINE void HTTPDocument::
set_persistent_connection(bool persistent_connection) {
_persistent_connection = persistent_connection;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_persistent_connection
// Access: Published
// Description: Returns whether the HTTPDocument should try to keep
// the connection to the server open and reuse that
// connection for multiple documents, or whether it
// should close the connection and open a new one for
// each request. See set_persistent_connection().
////////////////////////////////////////////////////////////////////
INLINE bool HTTPDocument::
get_persistent_connection() const {
return _persistent_connection;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_file_size
// Access: Published
@ -83,3 +181,34 @@ INLINE size_t HTTPDocument::
get_file_size() const {
return _file_size;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_document
// Access: Published
// Description: Opens the named document for reading, or if body is
// nonempty, posts data for a particular URL and
// retrieves the response.
////////////////////////////////////////////////////////////////////
INLINE bool HTTPDocument::
get_document(const URLSpec &url, const string &body) {
const char *method = "GET";
if (!body.empty()) {
method = "POST";
}
return send_request(method, url, body);
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_header
// Access: Published
// Description: Like get_document(), except only the header
// associated with the file is retrieved. This may be
// used to test for existence of the file; it might also
// return the size of the file (if the server gives us
// this information).
////////////////////////////////////////////////////////////////////
INLINE bool HTTPDocument::
get_header(const URLSpec &url) {
return send_request("HEAD", url, string());
}

840
panda/src/downloader/httpDocument.cxx
View File

@ -17,32 +17,148 @@
////////////////////////////////////////////////////////////////////
#include "httpDocument.h"
#include "httpClient.h"
#include "bioStream.h"
#include "chunkedStream.h"
#include "identityStream.h"
#ifdef HAVE_SSL
TypeHandle HTTPDocument::_type_handle;
static const char base64_table[64] = {
'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
'w', 'x', 'y', 'z', '0', '1', '2', '3',
'4', '5', '6', '7', '8', '9', '+', '/',
};
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::Constructor
// Access: Public
// Access: Private
// Description:
////////////////////////////////////////////////////////////////////
HTTPDocument::
HTTPDocument(BIO *bio, bool owns_bio) {
HTTPDocument(HTTPClient *client, BIO *bio) :
_client(client),
_bio(bio)
{
_owns_bio = false;
_source = (IBioStream *)NULL;
_persistent_connection = false;
_state = S_new;
_read_index = 0;
_file_size = 0;
_status_code = 0;
_status_string = "No connection";
_proxy = _client->get_proxy();
_http_version = _client->get_http_version();
_http_version_string = _client->get_http_version_string();
}
if (bio != (BIO *)NULL) {
_source = new IBioStream(bio, owns_bio);
read_headers();
determine_content_length();
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::send_request
// Access: Private
// Description: This is normally called immediately after
// construction to send the request to the server and
// read the response. It can't be called as part of the
// constructor because it may involve an up-and-down
// change in the reference count of the HTTPDocument
// object, which would inadvertently cause the object to
// be deleted if it hasn't returned from its constructor
// yet!
////////////////////////////////////////////////////////////////////
bool HTTPDocument::
send_request(const string &method, const URLSpec &url, const string &body) {
// Let's call this before we call make_header(), so we'll get the
// right HTTP version and proxy information etc.
set_url(url);
prepare_for_next();
} else {
_source = (IBioStream *)NULL;
_status_code = 0;
_status_string = "No connection";
string header;
make_header(header, method, url, body);
send_request(header, body);
if ((get_status_code() / 100) == 3 && get_status_code() != 305) {
// Redirect. Should we handle it automatically?
if (!get_redirect().empty() && (method == "GET" || method == "HEAD")) {
// Sure!
pset<URLSpec> already_seen;
bool keep_going;
do {
keep_going = false;
if (downloader_cat.is_debug()) {
downloader_cat.debug()
<< "following redirect to " << get_redirect() << "\n";
}
URLSpec new_url = get_redirect();
if (already_seen.insert(new_url).second) {
if (url.has_username()) {
new_url.set_username(url.get_username());
}
set_url(new_url);
if (prepare_for_next()) {
make_header(header, method, new_url, body);
send_request(header, body);
keep_going =
((get_status_code() / 100) == 3 && get_status_code() != 305);
}
}
} while (keep_going);
}
}
return is_valid();
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::send_request
// Access: Private
// Description: This is a lower-level interface than the above
// send_request(); it accepts a header and body string
// that have already been defined.
////////////////////////////////////////////////////////////////////
bool HTTPDocument::
send_request(const string &header, const string &body) {
if (prepare_for_next()) {
issue_request(header, body);
if (get_status_code() == 407 && !_proxy.empty()) {
// 407: not authorized to proxy. Try to get the authorization.
string authenticate_request = get_header_value("Proxy-Authenticate");
string authorization;
if (get_authorization(authorization, authenticate_request, _proxy, true)) {
string new_header = header;
new_header += "Proxy-Authorization: ";
new_header += authorization;
new_header += "\r\n";
if (prepare_for_next()) {
issue_request(new_header, body);
}
}
}
if (get_status_code() == 401) {
// 401: not authorized to remote server. Try to get the authorization.
string authenticate_request = get_header_value("WWW-Authenticate");
string authorization;
if (get_authorization(authorization, authenticate_request, _url, false)) {
string new_header = header;
new_header += "Authorization: ";
new_header += authorization;
new_header += "\r\n";
if (prepare_for_next()) {
issue_request(new_header, body);
}
}
}
}
return is_valid();
}
////////////////////////////////////////////////////////////////////
@ -52,10 +168,7 @@ HTTPDocument(BIO *bio, bool owns_bio) {
////////////////////////////////////////////////////////////////////
HTTPDocument::
~HTTPDocument() {
if (_source != (IBioStream *)NULL) {
delete _source;
_source = (IBioStream *)NULL;
}
free_bio();
}
////////////////////////////////////////////////////////////////////
@ -91,6 +204,31 @@ is_regular_file() const {
return is_valid();
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::will_close_connection
// Access: Public
// Description: Returns true if the server has indicated it will
// close the connection after this document has been
// read, or false if it will remain open (and future
// documents may be requested on the same connection).
////////////////////////////////////////////////////////////////////
bool HTTPDocument::
will_close_connection() const {
if (get_http_version() == HTTPClient::HV_10) {
// HTTP 1.0 always closes.
return true;
}
string connection = get_header_value("Connection");
if (downcase(connection) == "close") {
// The server says it will close.
return true;
}
// Assume the serve will keep it open.
return false;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::open_read_file
// Access: Public, Virtual
@ -102,24 +240,7 @@ is_regular_file() const {
////////////////////////////////////////////////////////////////////
istream *HTTPDocument::
open_read_file() const {
if (_source == (IBioStream *)NULL) {
return NULL;
}
string transfer_coding = get_header_value("Transfer-Encoding");
for (string::iterator si = transfer_coding.begin();
si != transfer_coding.end();
++si) {
(*si) = tolower(*si);
}
istream *result = _source;
if (transfer_coding == "chunked") {
result = new IChunkedStream(_source, true, (HTTPDocument *)this);
}
((HTTPDocument *)this)->_source = (IBioStream *)NULL;
return result;
return ((HTTPDocument *)this)->read_body(true);
}
////////////////////////////////////////////////////////////////////
@ -131,7 +252,7 @@ open_read_file() const {
////////////////////////////////////////////////////////////////////
string HTTPDocument::
get_header_value(const string &key) const {
Headers::const_iterator hi = _headers.find(key);
Headers::const_iterator hi = _headers.find(downcase(key));
if (hi != _headers.end()) {
return (*hi).second;
}
@ -153,15 +274,123 @@ write_headers(ostream &out) const {
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::read_headers
// Function: HTTPDocument::make_header
// Access: Private
// Description: Formats the appropriate GET or POST (or whatever)
// request to send to the server. Also saves the
// indicated url.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
make_header(string &header, const string &method,
const URLSpec &url, const string &body) {
set_url(url);
_method = method;
string path;
if (_proxy.empty()) {
path = _url.get_path();
} else {
URLSpec url_no_username = _url;
url_no_username.set_username(string());
path = url_no_username.get_url();
}
ostringstream stream;
stream
<< method << " " << path << " " << get_http_version_string() << "\r\n";
if (_http_version > HTTPClient::HV_10) {
stream
<< "Host: " << _url.get_server() << "\r\n";
if (!get_persistent_connection()) {
stream
<< "Connection: close\r\n";
}
}
if (!body.empty()) {
stream
<< "Content-Type: application/x-www-form-urlencoded\r\n"
<< "Content-Length: " << body.length() << "\r\n";
}
header = stream.str();
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::set_url
// Access: Private
// Description: Specifies the document's URL before attempting a
// connection. This controls the name of the server to
// be contacted, etc.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
set_url(const URLSpec &url) {
// If we change between http and https, we have to reset the
// connection regardless of proxy. Otherwise, we have to drop the
// connection if the server or port changes, unless we're
// communicating through a proxy.
if (url.get_scheme() != _url.get_scheme() ||
(_proxy.empty() && (url.get_server() != _url.get_server() ||
url.get_port() != _url.get_port()))) {
free_bio();
}
_url = url;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::issue_request
// Access: Private
// Description: Issues the request to the HTTP server and waits for a
// response.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
issue_request(const string &header, const string &body) {
if (_bio != (BIO *)NULL) {
string request = header;
request += "\r\n";
request += body;
#ifndef NDEBUG
if (downloader_cat.is_spam()) {
show_send(request);
}
#endif
BIO_puts(_bio, request.c_str());
read_http_response();
if (_source->eof() || _source->fail()) {
if (downloader_cat.is_debug()) {
downloader_cat.debug()
<< "Whoops, socket closed.\n";
free_bio();
if (prepare_for_next()) {
#ifndef NDEBUG
if (downloader_cat.is_spam()) {
show_send(request);
}
#endif
BIO_puts(_bio, request.c_str());
read_http_response();
}
}
}
}
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::read_http_response
// Access: Private
// Description: Reads all of the responses from the server up until
// the first blank line, and stores the list of header
// key:value pairs so retrieved.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
read_headers() {
read_http_response() {
nassertv(_source != (IBioStream *)NULL);
_headers.clear();
_realm = string();
// The first line back should include the HTTP version and the
// result code.
@ -170,9 +399,10 @@ read_headers() {
if (!line.empty() && line[line.length() - 1] == '\r') {
line = line.substr(0, line.length() - 1);
}
if (downloader_cat.is_debug()) {
downloader_cat.debug() << "recv: " << line << "\n";
if (downloader_cat.is_spam()) {
downloader_cat.spam() << "recv: " << line << "\n";
}
if (!(*_source) || line.length() < 5 || line.substr(0, 5) != "HTTP/") {
// Not an HTTP response.
_status_code = 0;
@ -185,7 +415,8 @@ read_headers() {
while (p < line.length() && !isspace(line[p])) {
p++;
}
_http_version = line.substr(0, p);
_http_version_string = line.substr(0, p);
_http_version = HTTPClient::parse_http_version_string(_http_version_string);
while (p < line.length() && isspace(line[p])) {
p++;
@ -211,9 +442,10 @@ read_headers() {
if (!line.empty() && line[line.length() - 1] == '\r') {
line = line.substr(0, line.length() - 1);
}
if (downloader_cat.is_debug()) {
downloader_cat.debug() << "recv: " << line << "\n";
if (downloader_cat.is_spam()) {
downloader_cat.spam() << "recv: " << line << "\n";
}
while (!_source->eof() && !_source->fail() && !line.empty()) {
if (isspace(line[0])) {
// If the line begins with a space, that continues the previous
@ -228,13 +460,13 @@ read_headers() {
// If the line does not begin with a space, that defines a new
// field.
if (!field_name.empty()) {
_headers[field_name] = field_value;
store_header_field(field_name, field_value);
field_value = string();
}
size_t colon = line.find(':');
if (colon != string::npos) {
field_name = line.substr(0, colon);
field_name = downcase(line.substr(0, colon));
p = colon + 1;
while (p < line.length() && isspace(line[p])) {
p++;
@ -247,31 +479,531 @@ read_headers() {
if (!line.empty() && line[line.length() - 1] == '\r') {
line = line.substr(0, line.length() - 1);
}
if (downloader_cat.is_debug()) {
downloader_cat.debug() << "recv: " << line << "\n";
if (downloader_cat.is_spam()) {
downloader_cat.spam() << "recv: " << line << "\n";
}
}
if (!field_name.empty()) {
_headers[field_name] = field_value;
store_header_field(field_name, field_value);
field_value = string();
}
// A blank line terminates the headers.
}
_state = S_read_header;
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::determine_content_length
// Access: Private
// Description: Determines the file size based on the Content-Length
// field if it has been supplied.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
determine_content_length() {
if (get_status_code() / 100 == 1 ||
get_status_code() == 204 ||
get_status_code() == 304 ||
_method == "HEAD") {
// These status codes, or method HEAD, indicate we have no body.
// Therefore, we have already read the (nonexistent) body.
_state = S_read_trailer;
}
_file_size = 0;
string content_length = get_header_value("Content-Length");
if (!content_length.empty()) {
_file_size = atoi(content_length.c_str());
}
_redirect = get_header_value("Location");
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::store_header_field
// Access: Private
// Description: Stores a single name: value pair in the header list,
// or appends the value to the end of the existing
// value, if the header has been repeated.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
store_header_field(const string &field_name, const string &field_value) {
pair<Headers::iterator, bool> insert_result =
_headers.insert(Headers::value_type(field_name, field_value));
if (!insert_result.second) {
// It didn't insert; thus, the field already existed. Append the
// new value.
Headers::iterator hi = insert_result.first;
(*hi).second += ", ";
(*hi).second += field_value;
}
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_authorization
// Access: Private
// Description: Looks for a username:password to satisfy the given
// authenticate_request string from the server or proxy.
// If found, fills in authorization and returns true;
// otherwise, returns false.
////////////////////////////////////////////////////////////////////
bool HTTPDocument::
get_authorization(string &authorization, const string &authenticate_request,
const URLSpec &url, bool is_proxy) {
AuthenticationSchemes schemes;
parse_authentication_schemes(schemes, authenticate_request);
AuthenticationSchemes::iterator si;
si = schemes.find("basic");
if (si != schemes.end()) {
return get_basic_authorization(authorization, (*si).second, url, is_proxy);
}
downloader_cat.warning()
<< "Don't know how to use any of the server's available authorization schemes:\n";
for (si = schemes.begin(); si != schemes.end(); ++si) {
downloader_cat.warning() << (*si).first << "\n";
}
return false;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::downcase
// Access: Private, Static
// Description: Returns the input string with all uppercase letters
// converted to lowercase.
////////////////////////////////////////////////////////////////////
string HTTPDocument::
downcase(const string &s) {
string result;
result.reserve(s.size());
string::const_iterator p;
for (p = s.begin(); p != s.end(); ++p) {
result += tolower(*p);
}
return result;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::base64_encode
// Access: Private, Static
// Description: Returns the input string encoded using base64. No
// respect is paid to maintaining a 76-char line length.
////////////////////////////////////////////////////////////////////
string HTTPDocument::
base64_encode(const string &s) {
// Collect the string 3 bytes at a time into 24-bit words, then
// output each word using 4 bytes.
size_t num_words = (s.size() + 2) / 3;
string result;
result.reserve(num_words * 4);
size_t p;
for (p = 0; p + 2 < s.size(); p += 3) {
unsigned int word =
((unsigned)s[p] << 16) |
((unsigned)s[p + 1] << 8) |
((unsigned)s[p + 2]);
result += base64_table[(word >> 18) & 0x3f];
result += base64_table[(word >> 12) & 0x3f];
result += base64_table[(word >> 6) & 0x3f];
result += base64_table[(word) & 0x3f];
}
// What's left over?
if (p < s.size()) {
unsigned int word = ((unsigned)s[p] << 16);
p++;
if (p < s.size()) {
word |= ((unsigned)s[p] << 8);
p++;
nassertr(p == s.size(), result);
result += base64_table[(word >> 18) & 0x3f];
result += base64_table[(word >> 12) & 0x3f];
result += base64_table[(word >> 6) & 0x3f];
result += '=';
} else {
result += base64_table[(word >> 18) & 0x3f];
result += base64_table[(word >> 12) & 0x3f];
result += '=';
result += '=';
}
}
return result;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::scan_quoted_or_unquoted_string
// Access: Private, Static
// Description: Scans the string source beginning at character
// position start, to identify either the
// (spaced-delimited) unquoted string there, or the
// (quote-delimited) quoted string. In either case,
// fills the string found into result, and returns the
// next character position after the string (or after
// its closing quote mark).
////////////////////////////////////////////////////////////////////
size_t HTTPDocument::
scan_quoted_or_unquoted_string(string &result, const string &source,
size_t start) {
result = string();
if (start < source.length()) {
if (source[start] == '"') {
// Quoted string.
size_t p = start + 1;
while (p < source.length() && source[p] != '"') {
if (source[p] == '\\') {
// Backslash escapes.
++p;
if (p < source.length()) {
result += source[p];
++p;
}
} else {
result += source[p];
++p;
}
}
if (p < source.length()) {
++p;
}
return p;
}
// Unquoted string.
size_t p = start;
while (p < source.length() && source[p] != ',' && !isspace(source[p])) {
result += source[p];
++p;
}
return p;
}
// Empty string.
return start;
}
#ifndef NDEBUG
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::show_send
// Access: Private, Static
// Description: Writes the outgoing message, one line at a time, to
// the debugging log.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
show_send(const string &message) {
size_t start = 0;
size_t newline = message.find('\n', start);
while (newline != string::npos) {
downloader_cat.spam()
<< "send: " << message.substr(start, newline - start + 1);
start = newline + 1;
newline = message.find('\n', start);
}
if (start < message.length()) {
downloader_cat.spam()
<< "send: " << message.substr(start) << " (no newline)\n";
}
}
#endif // NDEBUG
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::parse_authentication_schemes
// Access: Private, Static
// Description: Decodes the text following a WWW-Authenticate: or
// Proxy-Authenticate: header field.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
parse_authentication_schemes(HTTPDocument::AuthenticationSchemes &schemes,
const string &field_value) {
// This string will consist of one or more records of the form:
//
// scheme token=value[,token=value[,...]]
//
// If there are multiple records, they will be comma-delimited,
// which makes parsing just a bit tricky.
// Start by skipping initial whitespace.
size_t p = 0;
while (p < field_value.length() && isspace(field_value[p])) {
++p;
}
if (p < field_value.length()) {
size_t q = p;
while (q < field_value.length() && !isspace(field_value[q])) {
++q;
}
// Here's our first scheme.
string scheme = downcase(field_value.substr(p, q - p));
Tokens *tokens = &(schemes[scheme]);
// Now pull off the tokens, one at a time.
p = q + 1;
while (p < field_value.length()) {
q = p;
while (q < field_value.length() && field_value[q] != '=' &&
field_value[q] != ',' && !isspace(field_value[q])) {
++q;
}
if (field_value[q] == '=') {
// This is a token.
string token = downcase(field_value.substr(p, q - p));
string value;
p = scan_quoted_or_unquoted_string(value, field_value, q + 1);
(*tokens)[token] = value;
// Skip trailing whitespace and extra commas.
while (p < field_value.length() &&
(field_value[p] == ',' || isspace(field_value[p]))) {
++p;
}
} else {
// This is not a token; it must be the start of a new scheme.
scheme = downcase(field_value.substr(p, q - p));
tokens = &(schemes[scheme]);
p = q + 1;
}
}
}
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::read_body
// Access: Private
// Description: Returns a newly-allocated istream suitable for
// reading the body of the document. If owns_source is
// true, the ownership of the _source pointer will be
// passed to the istream; otherwise, it will be
// retained. (owns_source must be true in order to read
// "identity" encoded documents.)
////////////////////////////////////////////////////////////////////
istream *HTTPDocument::
read_body(bool owns_source) {
if (_state != S_read_header || _source == (IBioStream *)NULL) {
return NULL;
}
string transfer_coding = downcase(get_header_value("Transfer-Encoding"));
string content_length = get_header_value("Content-Length");
istream *result;
if (transfer_coding == "chunked") {
// Chunked can be used directly.
_file_size = 0;
_state = S_started_body;
_read_index++;
result = new IChunkedStream(_source, owns_source, (HTTPDocument *)this);
if (owns_source) {
_source = (IBioStream *)NULL;
}
} else if (!content_length.empty()) {
// If we have a content length, we can use an IdentityStream.
_state = S_started_body;
_read_index++;
result = new IIdentityStream(_source, owns_source, (HTTPDocument *)this, _file_size);
if (owns_source) {
_source = (IBioStream *)NULL;
}
} else if (owns_source) {
// If we own the source, we can return it.
_state = S_started_body;
result = _source;
_source = (IBioStream *)NULL;
} else {
// Otherwise, we don't own the source; too bad.
result = NULL;
}
return result;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::prepare_for_next
// Access: Private
// Description: Resets the state to prepare it for sending a new
// request to the server. This might mean closing the
// connection and opening a new one, or it might mean
// skipping past the unread body in the persistent
// connection, or it might do nothing at all if the body
// has already been completely read.
////////////////////////////////////////////////////////////////////
bool HTTPDocument::
prepare_for_next() {
if (get_persistent_connection() && !will_close_connection() &&
_proxy == _client->get_proxy()) {
// See if we can reuse the current connection.
if (_state == S_read_header) {
// We have read the header; now skip past the body.
istream *body = read_body(false);
if (body != (istream *)NULL) {
string line;
getline(*body, line);
while (!body->fail() && !body->eof()) {
if (downloader_cat.is_spam()) {
downloader_cat.spam() << "skip: " << line << "\n";
}
getline(*body, line);
}
nassertr(body != _source, false);
delete body;
}
}
if (_source == (IBioStream *)NULL) {
_source = new IBioStream(_bio, false);
}
if (_state == S_read_body) {
// We have read the body, but there's a trailer to read.
string line;
getline(*_source, line);
if (!line.empty() && line[line.length() - 1] == '\r') {
line = line.substr(0, line.length() - 1);
}
if (downloader_cat.is_spam()) {
downloader_cat.spam() << "skip: " << line << "\n";
}
while (!_source->eof() && !_source->fail() && !line.empty()) {
getline(*_source, line);
if (!line.empty() && line[line.length() - 1] == '\r') {
line = line.substr(0, line.length() - 1);
}
if (downloader_cat.is_spam()) {
downloader_cat.spam() << "skip: " << line << "\n";
}
}
_state = S_read_trailer;
}
if (_state == S_read_trailer) {
// Great; this connection is ready to go!
return true;
}
}
if (_bio != (BIO *)NULL && _state == S_new) {
// If we have a BIO and the _state is S_new, then we haven't done
// anything with the BIO yet, so we can still use it.
if (_source == (IBioStream *)NULL) {
_source = new IBioStream(_bio, false);
}
return true;
}
// Either the client will close the connection after reading the
// body, or we were only partly through reading the body elsewhere;
// or possibly we don't have a connection yet at all. In any case,
// we must now get a new connection.
if (_bio != (BIO *)NULL && !_owns_bio) {
// We have a connection, but we don't own it, so we can't close
// it. Too bad.
return false;
}
// Go ahead and close the old BIO.
free_bio();
_proxy = _client->get_proxy();
_http_version = _client->get_http_version();
_http_version_string = _client->get_http_version_string();
_bio = _client->establish_connection(_url);
_owns_bio = true;
if (_bio != (BIO *)NULL) {
_source = new IBioStream(_bio, false);
return true;
}
return false;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::free_bio
// Access: Private
// Description: Frees the BIO and its IBioStream object, if
// allocated. This will close the connection if it is
// open.
////////////////////////////////////////////////////////////////////
void HTTPDocument::
free_bio() {
if (_source != (IBioStream *)NULL) {
delete _source;
_source = (IBioStream *)NULL;
}
if (_bio != (BIO *)NULL) {
if (_owns_bio) {
// TODO: We should be more careful here to manage reference
// counts so we don't free the bio out from under a BIOStreamBuf
// that's trying to read from it.
if (downloader_cat.is_debug()) {
const URLSpec &url = _proxy.empty() ? _url : _proxy;
downloader_cat.debug()
<< "Dropping connection to " << url.get_server() << "\n";
}
BIO_free_all(_bio);
}
_bio = (BIO *)NULL;
}
_read_index++;
_state = S_new;
}
////////////////////////////////////////////////////////////////////
// Function: HTTPDocument::get_basic_authorization
// Access: Private
// Description: Looks for a username:password to satisfy the "Basic"
// scheme authorization request from the server or
// proxy.
////////////////////////////////////////////////////////////////////
bool HTTPDocument::
get_basic_authorization(string &authorization, const HTTPDocument::Tokens &tokens, const URLSpec &url, bool is_proxy) {
Tokens::const_iterator ti;
ti = tokens.find("realm");
if (ti != tokens.end()) {
_realm = (*ti).second;
}
string username;
// Look in several places in order to find the matching username.
// Fist, if there's a username on the URL, that always wins.
if (url.has_username()) {
username = url.get_username();
}
// Otherwise, start looking on the HTTPClient.
if (is_proxy) {
if (username.empty()) {
// Try the *proxy/realm.
username = _client->get_username("*proxy", _realm);
}
if (username.empty()) {
// Then, try *proxy/any realm.
username = _client->get_username("*proxy", string());
}
}
if (username.empty()) {
// Try the specific server/realm.
username = _client->get_username(url.get_server(), _realm);
}
if (username.empty()) {
// Then, try the specific server/any realm.
username = _client->get_username(url.get_server(), string());
}
if (username.empty()) {
// Then, try any server with this realm.
username = _client->get_username(string(), _realm);
}
if (username.empty()) {
// Then, take the general password.
username = _client->get_username(string(), string());
}
if (username.empty()) {
// No username:password available.
return false;
}
authorization = "Basic " + base64_encode(username);
return true;
}
#endif // HAVE_SSL

78
panda/src/downloader/httpDocument.h
View File

@ -28,20 +28,28 @@
#ifdef HAVE_SSL
#include "httpClient.h"
#include "urlSpec.h"
#include "virtualFile.h"
#include "pmap.h"
#include <openssl/ssl.h>
class IBioStream;
class HTTPClient;
////////////////////////////////////////////////////////////////////
// Class : HTTPDocument
// Description : A single document retrieved from an HTTP server.
////////////////////////////////////////////////////////////////////
class EXPCL_PANDAEXPRESS HTTPDocument : public VirtualFile {
private:
HTTPDocument(HTTPClient *client, BIO *bio = NULL);
bool send_request(const string &method, const URLSpec &url,
const string &body);
bool send_request(const string &header, const string &body);
public:
HTTPDocument(BIO *bio, bool owns_bio);
virtual ~HTTPDocument();
virtual VirtualFileSystem *get_file_system() const;
@ -50,32 +58,90 @@ public:
virtual bool is_regular_file() const;
virtual istream *open_read_file() const;
bool will_close_connection() const;
PUBLISHED:
INLINE bool is_valid() const;
INLINE const string &get_http_version() const;
INLINE const URLSpec &get_url() const;
INLINE HTTPClient::HTTPVersion get_http_version() const;
INLINE const string &get_http_version_string() const;
INLINE int get_status_code() const;
INLINE const string &get_status_string() const;
INLINE const string &get_realm() const;
INLINE const URLSpec &get_redirect() const;
string get_header_value(const string &key) const;
INLINE void set_persistent_connection(bool persistent_connection);
INLINE bool get_persistent_connection() const;
INLINE size_t get_file_size() const;
void write_headers(ostream &out) const;
INLINE bool get_document(const URLSpec &url, const string &body = string());
INLINE bool get_header(const URLSpec &url);
private:
void read_headers();
void determine_content_length();
void make_header(string &header, const string &method,
const URLSpec &url, const string &body);
void set_url(const URLSpec &url);
void issue_request(const string &header, const string &body);
void read_http_response();
void store_header_field(const string &field_name, const string &field_value);
bool get_authorization(string &authorization,
const string &authenticate_request,
const URLSpec &url, bool is_proxy);
static string downcase(const string &s);
static string base64_encode(const string &s);
static size_t scan_quoted_or_unquoted_string(string &result, const string &source, size_t start);
#ifndef NDEBUG
static void show_send(const string &message);
#endif
istream *read_body(bool owns_source);
bool prepare_for_next();
void free_bio();
HTTPClient *_client;
URLSpec _proxy;
BIO *_bio;
bool _owns_bio;
IBioStream *_source;
bool _persistent_connection;
string _http_version;
URLSpec _url;
string _method;
enum State {
S_new,
S_read_header,
S_started_body,
S_read_body,
S_read_trailer
};
State _state;
int _read_index;
HTTPClient::HTTPVersion _http_version;
string _http_version_string;
int _status_code;
string _status_string;
string _realm;
URLSpec _redirect;
typedef pmap<string, string> Headers;
Headers _headers;
size_t _file_size;
typedef pmap<string, string> Tokens;
typedef pmap<string, Tokens> AuthenticationSchemes;
static void parse_authentication_schemes(AuthenticationSchemes &schemes,
const string &field_value);
bool get_basic_authorization(string &authorization, const Tokens &tokens,
const URLSpec &url, bool is_proxy);
public:
virtual TypeHandle get_type() const {
@ -94,6 +160,8 @@ public:
private:
static TypeHandle _type_handle;
friend class ChunkedStreamBuf;
friend class IdentityStreamBuf;
friend class HTTPClient;
};
#include "httpDocument.I"

63
panda/src/downloader/identityStream.I Normal file
View File

@ -0,0 +1,63 @@
// Filename: identityStream.I
// Created by: drose (09Oct02)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license. You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////
// Function: IIdentityStream::Constructor
// Access: Public
// Description:
////////////////////////////////////////////////////////////////////
INLINE IIdentityStream::
IIdentityStream() : istream(&_buf) {
}
////////////////////////////////////////////////////////////////////
// Function: IIdentityStream::Constructor
// Access: Public
// Description:
////////////////////////////////////////////////////////////////////
INLINE IIdentityStream::
IIdentityStream(istream *source, bool owns_source,
HTTPDocument *doc, size_t content_length) : istream(&_buf) {
open(source, owns_source, doc, content_length);
}
////////////////////////////////////////////////////////////////////
// Function: IIdentityStream::open
// Access: Public
// Description:
////////////////////////////////////////////////////////////////////
INLINE IIdentityStream &IIdentityStream::
open(istream *source, bool owns_source, HTTPDocument *doc,
size_t content_length) {
clear(0);
_buf.open_read(source, owns_source, doc, content_length);
return *this;
}
////////////////////////////////////////////////////////////////////
// Function: IIdentityStream::close
// Access: Public
// Description: Resets the IdentityStream to empty, but does not actually
// close the source IDENTITY unless owns_source was true.
////////////////////////////////////////////////////////////////////
INLINE IIdentityStream &IIdentityStream::
close() {
_buf.close_read();
return *this;
}

19
panda/src/downloader/identityStream.cxx Normal file
View File

@ -0,0 +1,19 @@
// Filename: identityStream.cxx
// Created by: drose (09Oct02)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license. You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////
#include "identityStream.h"

60
panda/src/downloader/identityStream.h Normal file
View File

@ -0,0 +1,60 @@
// Filename: identityStream.h
// Created by: drose (09Oct02)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license. You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////
#ifndef IDENTITYSTREAM_H
#define IDENTITYSTREAM_H
#include "pandabase.h"
#include "identityStreamBuf.h"
class HTTPDocument;
////////////////////////////////////////////////////////////////////
// Class : IIdentityStream
// Description : An input stream object that reads data from a source
// istream, but automatically decodes the "identity"
// transfer-coding specified by an HTTP server.
//
// In practice, this just means it reads from the sub
// stream (like a SubStreamBuf) up to but not past the
// specified content-length. (If the content-length was
// unspecified, this class cannot be used.) It also
// updates the HTTPDocument when the stream is
// completely read.
////////////////////////////////////////////////////////////////////
// No need to export from DLL.
class IIdentityStream : public istream {
public:
INLINE IIdentityStream();
INLINE IIdentityStream(istream *source, bool owns_source,
HTTPDocument *doc, size_t content_length);
INLINE IIdentityStream &open(istream *source, bool owns_source,
HTTPDocument *doc, size_t content_length);
INLINE IIdentityStream &close();
private:
IdentityStreamBuf _buf;
};
#include "identityStream.I"
#endif

160
panda/src/downloader/identityStreamBuf.cxx Normal file
View File

@ -0,0 +1,160 @@
// Filename: identityStreamBuf.cxx
// Created by: drose (09Oct02)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license. You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////
#include "identityStreamBuf.h"
#ifndef HAVE_STREAMSIZE
// Some compilers (notably SGI) don't define this for us
typedef int streamsize;
#endif /* HAVE_STREAMSIZE */
////////////////////////////////////////////////////////////////////
// Function: IdentityStreamBuf::Constructor
// Access: Public
// Description:
////////////////////////////////////////////////////////////////////
IdentityStreamBuf::
IdentityStreamBuf() {
_source = (istream *)NULL;
_owns_source = false;
_bytes_remaining = 0;
#ifdef WIN32_VC
// In spite of the claims of the MSDN Library to the contrary,
// Windows doesn't seem to provide an allocate() function, so we'll
// do it by hand.
char *buf = new char[4096];
char *ebuf = buf + 4096;
setg(buf, ebuf, ebuf);
setp(buf, ebuf);
#else
allocate();
setg(base(), ebuf(), ebuf());
setp(base(), ebuf());
#endif
}
////////////////////////////////////////////////////////////////////
// Function: IdentityStreamBuf::Destructor
// Access: Public, Virtual
// Description:
////////////////////////////////////////////////////////////////////
IdentityStreamBuf::
~IdentityStreamBuf() {
close_read();
}
////////////////////////////////////////////////////////////////////
// Function: IdentityStreamBuf::open_read
// Access: Public
// Description: If the document pointer is non-NULL, it will be
// updated with the length of the file as it is derived
// from the identity encoding.
////////////////////////////////////////////////////////////////////
void IdentityStreamBuf::
open_read(istream *source, bool owns_source, HTTPDocument *doc,
size_t content_length) {
_source = source;
_owns_source = owns_source;
_doc = doc;
_bytes_remaining = content_length;
if (_doc != (HTTPDocument *)NULL) {
_read_index = doc->_read_index;
}
}
////////////////////////////////////////////////////////////////////
// Function: IdentityStreamBuf::close_read
// Access: Public
// Description:
////////////////////////////////////////////////////////////////////
void IdentityStreamBuf::
close_read() {
if (_source != (istream *)NULL) {
if (_owns_source) {
delete _source;
_owns_source = false;
}
_source = (istream *)NULL;
}
}
////////////////////////////////////////////////////////////////////
// Function: IdentityStreamBuf::underflow
// Access: Protected, Virtual
// Description: Called by the system istream implementation when its
// internal buffer needs more characters.
////////////////////////////////////////////////////////////////////
int IdentityStreamBuf::
underflow() {
// Sometimes underflow() is called even if the buffer is not empty.
if (gptr() >= egptr()) {
size_t buffer_size = egptr() - eback();
gbump(-(int)buffer_size);
size_t num_bytes = buffer_size;
size_t read_count = read_chars(gptr(), buffer_size);
if (read_count != num_bytes) {
// Oops, we didn't read what we thought we would.
if (read_count == 0) {
return EOF;
}
// Slide what we did read to the top of the buffer.
nassertr(read_count < num_bytes, EOF);
size_t delta = num_bytes - read_count;
memmove(gptr() + delta, gptr(), read_count);
gbump(delta);
}
}
return (unsigned char)*gptr();
}
////////////////////////////////////////////////////////////////////
// Function: IdentityStreamBuf::read_chars
// Access: Private
// Description: Gets some characters from the source stream.
////////////////////////////////////////////////////////////////////
size_t IdentityStreamBuf::
read_chars(char *start, size_t length) {
if (_bytes_remaining == 0) {
return 0;
}
// Extract some of the bytes remaining in the chunk.
length = min(length, _bytes_remaining);
_source->read(start, length);
length = _source->gcount();
_bytes_remaining -= length;
if (_bytes_remaining == 0) {
// We're done.
if (_doc != (HTTPDocument *)NULL && _read_index == _doc->_read_index) {
// An IdentityStreamBuf doesn't have a trailer, so we've already
// "read" it.
_doc->_state = HTTPDocument::S_read_trailer;
}
}
return length;
}

54
panda/src/downloader/identityStreamBuf.h Normal file
View File

@ -0,0 +1,54 @@
// Filename: identityStreamBuf.h
// Created by: drose (09Oct02)
//
////////////////////////////////////////////////////////////////////
//
// PANDA 3D SOFTWARE
// Copyright (c) 2001, Disney Enterprises, Inc. All rights reserved
//
// All use of this software is subject to the terms of the Panda 3d
// Software license. You should have received a copy of this license
// along with this source code; you will also find a current copy of
// the license at http://www.panda3d.org/license.txt .
//
// To contact the maintainers of this program write to
// panda3d@yahoogroups.com .
//
////////////////////////////////////////////////////////////////////
#ifndef IDENTITYSTREAMBUF_H
#define IDENTITYSTREAMBUF_H
#include "pandabase.h"
#include "httpDocument.h"
#include "pointerTo.h"
////////////////////////////////////////////////////////////////////
// Class : IdentityStreamBuf
// Description : The streambuf object that implements
// IIdentityStream.
////////////////////////////////////////////////////////////////////
class EXPCL_PANDAEXPRESS IdentityStreamBuf : public streambuf {
public:
IdentityStreamBuf();
virtual ~IdentityStreamBuf();
void open_read(istream *source, bool owns_source, HTTPDocument *doc,
size_t content_length);
void close_read();
protected:
virtual int underflow(void);
private:
size_t read_chars(char *start, size_t length);
istream *_source;
bool _owns_source;
size_t _bytes_remaining;
PT(HTTPDocument) _doc;
int _read_index;
};
#endif

30
panda/src/downloader/urlSpec.I
View File

@ -47,6 +47,36 @@ operator = (const string &url) {
set_url(url);
}
////////////////////////////////////////////////////////////////////
// Function: URLSpec::Operator ==
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE bool URLSpec::
operator == (const URLSpec &other) const {
return _url == other._url;
}
////////////////////////////////////////////////////////////////////
// Function: URLSpec::Operator !=
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE bool URLSpec::
operator != (const URLSpec &other) const {
return !operator == (other);
}
////////////////////////////////////////////////////////////////////
// Function: URLSpec::Operator <
// Access: Published
// Description:
////////////////////////////////////////////////////////////////////
INLINE bool URLSpec::
operator < (const URLSpec &other) const {
return _url < other._url;
}
////////////////////////////////////////////////////////////////////
// Function: URLSpec::has_scheme
// Access: Published

4
panda/src/downloader/urlSpec.h
View File

@ -38,6 +38,10 @@ PUBLISHED:
INLINE void operator = (const string &url);
void operator = (const URLSpec &copy);
INLINE bool operator == (const URLSpec &other) const;
INLINE bool operator != (const URLSpec &other) const;
INLINE bool operator < (const URLSpec &other) const;
INLINE bool has_scheme() const;
INLINE bool has_authority() const;
INLINE bool has_username() const;