From 382ab2279b62d3635465c55caa97819b26fe43b2 Mon Sep 17 00:00:00 2001 From: David Rose Date: Thu, 17 Oct 2002 17:12:40 +0000 Subject: [PATCH] subdocument --- panda/src/downloader/httpChannel.I | 81 ++++++++++++++++++++----- panda/src/downloader/httpChannel.cxx | 90 +++++++++++++++++++++++++--- panda/src/downloader/httpChannel.h | 18 ++++-- 3 files changed, 162 insertions(+), 27 deletions(-) diff --git a/panda/src/downloader/httpChannel.I b/panda/src/downloader/httpChannel.I index 435925621b..c43da782b1 100644 --- a/panda/src/downloader/httpChannel.I +++ b/panda/src/downloader/httpChannel.I @@ -182,7 +182,7 @@ get_persistent_connection() const { // the data. // // This only has effect on the nonblocking I/O methods -// like request_document(), etc. The blocking methods +// like begin_document(), etc. The blocking methods // like get_document() always use as much CPU and // bandwidth as they can get. //////////////////////////////////////////////////////////////////// @@ -283,7 +283,7 @@ get_file_size() const { //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: post_form(const URLSpec &url, const string &body) { - begin_request("POST", url, body, false); + begin_request("POST", url, body, false, 0, 0); run(); return is_valid(); } @@ -296,7 +296,24 @@ post_form(const URLSpec &url, const string &body) { //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: get_document(const URLSpec &url) { - begin_request("GET", url, string(), false); + begin_request("GET", url, string(), false, 0, 0); + run(); + return is_valid(); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::get_subdocument +// Access: Published +// Description: Retrieves only the specified byte range of the +// indicated document. If last_byte is 0, it stands for +// the last byte of the document. When a subdocument is +// requested, get_file_size() and get_bytes_downloaded() +// will report the number of bytes of the subdocument, +// not of the complete document. +//////////////////////////////////////////////////////////////////// +INLINE bool HTTPChannel:: +get_subdocument(const URLSpec &url, size_t first_byte, size_t last_byte) { + begin_request("GET", url, string(), false, first_byte, last_byte); run(); return is_valid(); } @@ -312,17 +329,17 @@ get_document(const URLSpec &url) { //////////////////////////////////////////////////////////////////// INLINE bool HTTPChannel:: get_header(const URLSpec &url) { - begin_request("HEAD", url, string(), false); + begin_request("HEAD", url, string(), false, 0, 0); run(); return is_valid(); } //////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::request_post_form +// Function: HTTPChannel::begin_post_form // Access: Published // Description: Posts form data to a particular URL and retrieves the // response, all using non-blocking I/O. See -// request_document() and post_form(). +// begin_document() and post_form(). // // It is important to note that you *must* call run() // repeatedly after calling this method until run() @@ -332,12 +349,12 @@ get_header(const URLSpec &url) { // may not get posted. //////////////////////////////////////////////////////////////////// INLINE void HTTPChannel:: -request_post_form(const URLSpec &url, const string &body) { - begin_request("POST", url, body, true); +begin_post_form(const URLSpec &url, const string &body) { + begin_request("POST", url, body, true, 0, 0); } //////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::request_document +// Function: HTTPChannel::begin_document // Access: Published // Description: Begins a non-blocking request to retrieve a given // document. This method will return immediately, even @@ -351,19 +368,51 @@ request_post_form(const URLSpec &url, const string &body) { // is discarded. //////////////////////////////////////////////////////////////////// INLINE void HTTPChannel:: -request_document(const URLSpec &url) { - begin_request("GET", url, string(), true); +begin_document(const URLSpec &url) { + begin_request("GET", url, string(), true, 0, 0); } //////////////////////////////////////////////////////////////////// -// Function: HTTPChannel::request_header +// Function: HTTPChannel::begin_subdocument // Access: Published -// Description: Begins a non-blocking request to retrieve a given -// header. See request_document() and get_header(). +// Description: Begins a non-blocking request to retrieve only the +// specified byte range of the indicated document. If +// last_byte is 0, it stands for the last byte of the +// document. When a subdocument is requested, +// get_file_size() and get_bytes_downloaded() will +// report the number of bytes of the subdocument, not of +// the complete document. //////////////////////////////////////////////////////////////////// INLINE void HTTPChannel:: -request_header(const URLSpec &url) { - begin_request("HEAD", url, string(), true); +begin_subdocument(const URLSpec &url, size_t first_byte, size_t last_byte) { + begin_request("GET", url, string(), true, first_byte, last_byte); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::begin_header +// Access: Published +// Description: Begins a non-blocking request to retrieve a given +// header. See begin_document() and get_header(). +//////////////////////////////////////////////////////////////////// +INLINE void HTTPChannel:: +begin_header(const URLSpec &url) { + begin_request("HEAD", url, string(), true, 0, 0); +} + +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::get_bytes_downloaded +// Access: Published +// Description: Returns the number of bytes downloaded during the +// last (or current) download_to_file() or +// download_to_ram operation(). This can be used in +// conjunction with get_file_size() to report the +// percent complete (but be careful, since +// get_file_size() may return 0 if the server has not +// told us the size of the file). +//////////////////////////////////////////////////////////////////// +INLINE size_t HTTPChannel:: +get_bytes_downloaded() const { + return _bytes_downloaded; } //////////////////////////////////////////////////////////////////// diff --git a/panda/src/downloader/httpChannel.cxx b/panda/src/downloader/httpChannel.cxx index acbd738c7a..42c5d654e8 100644 --- a/panda/src/downloader/httpChannel.cxx +++ b/panda/src/downloader/httpChannel.cxx @@ -56,8 +56,11 @@ HTTPChannel(HTTPClient *client) : _max_updates_per_second = 1.0f / _seconds_per_update; _bytes_per_update = int(_max_bytes_per_second * _seconds_per_update); _nonblocking = false; + _first_byte = 0; + _last_byte = 0; _read_index = 0; _file_size = 0; + _bytes_downloaded = 0; _status_code = 0; _status_string = string(); _proxy = _client->get_proxy(); @@ -393,7 +396,7 @@ read_body() { // Description: Specifies the name of a file to download the // resulting document to. This should be called // immediately after get_document() or -// request_document() or related functions. +// begin_document() or related functions. // // In the case of the blocking I/O methods like // get_document(), this function will download the @@ -401,7 +404,7 @@ read_body() { // successfully downloaded, false otherwise. // // In the case of non-blocking I/O methods like -// request_document(), this function simply indicates an +// begin_document(), this function simply indicates an // intention to download to the indicated file. It // returns true if the file can be opened for writing, // false otherwise, but the contents will not be @@ -440,7 +443,7 @@ download_to_file(const Filename &filename) { // Description: Specifies a Ramfile object to download the // resulting document to. This should be called // immediately after get_document() or -// request_document() or related functions. +// begin_document() or related functions. // // In the case of the blocking I/O methods like // get_document(), this function will download the @@ -448,7 +451,7 @@ download_to_file(const Filename &filename) { // was successfully downloaded, false otherwise. // // In the case of non-blocking I/O methods like -// request_document(), this function simply indicates an +// begin_document(), this function simply indicates an // intention to download to the indicated Ramfile. It // returns true if the file can be opened for writing, // false otherwise, but the contents will not be @@ -859,10 +862,22 @@ run_reading_header() { _realm = string(); // Look for key properties in the header fields. + if (get_status_code() == 206) { + string content_range = get_header_value("Content-Range"); + if (!content_range.empty()) { + parse_content_range(content_range); + } + } + _file_size = 0; string content_length = get_header_value("Content-Length"); if (!content_length.empty()) { _file_size = atoi(content_length.c_str()); + + } else if (get_status_code() == 206 && _last_byte != 0) { + // Well, we didn't get a content-length from the server, but we + // can infer the number of bytes based on the range we requested. + _file_size = _last_byte - _first_byte + 1; } _redirect = get_header_value("Location"); @@ -1126,7 +1141,8 @@ run_download_to_file() { int ch = _body_stream->get(); while (!_body_stream->eof() && !_body_stream->fail()) { _download_to_file.put(ch); - if (do_throttle && ++count > _bytes_per_update) { + _bytes_downloaded++; + if (do_throttle && (++count > _bytes_per_update)) { // That's enough for now. return true; } @@ -1169,7 +1185,8 @@ run_download_to_ram() { int ch = _body_stream->get(); while (!_body_stream->eof() && !_body_stream->fail()) { _download_to_ramfile->_data += (char)ch; - if (do_throttle && ++count > _bytes_per_update) { + _bytes_downloaded++; + if (do_throttle && (++count > _bytes_per_update)) { // That's enough for now. return true; } @@ -1196,12 +1213,14 @@ run_download_to_ram() { //////////////////////////////////////////////////////////////////// void HTTPChannel:: begin_request(const string &method, const URLSpec &url, const string &body, - bool nonblocking) { + bool nonblocking, size_t first_byte, size_t last_byte) { reset_download_to(); _status_code = 0; _status_string = string(); _redirect_trail.clear(); _last_status_code = 0; + _file_size = 0; + _bytes_downloaded = 0; // Changing the proxy, or the nonblocking state, is grounds for // dropping the old connection, if any. @@ -1218,6 +1237,8 @@ begin_request(const string &method, const URLSpec &url, const string &body, _method = method; set_url(url); _body = body; + _first_byte = first_byte; + _last_byte = last_byte; make_header(); make_request_text(string()); @@ -1448,6 +1469,52 @@ parse_http_header() { return false; } +//////////////////////////////////////////////////////////////////// +// Function: HTTPChannel::parse_content_range +// Access: Private +// Description: Interprets the "Content-Range" header in the reply, +// and fills in _first_byte and _last_byte appropriately +// if the header response can be understood. +//////////////////////////////////////////////////////////////////// +bool HTTPChannel:: +parse_content_range(const string &content_range) { + // First, get the units indication. + size_t p = 0; + while (p < content_range.length() && !isspace(content_range[p])) { + p++; + } + + string units = content_range.substr(0, p); + while (p < content_range.length() && isspace(content_range[p])) { + p++; + } + + if (units == "bytes") { + const char *c_str = content_range.c_str(); + char *endptr; + if (p < content_range.length() && isdigit(content_range[p])) { + long first_byte = strtol(c_str + p, &endptr, 10); + p = endptr - c_str; + if (p < content_range.length() && content_range[p] == '-') { + p++; + if (p < content_range.length() && isdigit(content_range[p])) { + long last_byte = strtol(c_str + p, &endptr, 10); + p = endptr - c_str; + + if (last_byte >= first_byte) { + _first_byte = first_byte; + _last_byte = last_byte; + return true; + } + } + } + } + } + + // Invalid or unhandled response. + return false; +} + //////////////////////////////////////////////////////////////////// // Function: HTTPChannel::verify_server @@ -1772,6 +1839,15 @@ make_header() { } } + if (_last_byte != 0) { + stream + << "Range: bytes=" << _first_byte << "-" << _last_byte << "\r\n"; + + } else if (_first_byte != 0) { + stream + << "Range: bytes=" << _first_byte << "-\r\n"; + } + if (!_body.empty()) { stream << "Content-Type: application/x-www-form-urlencoded\r\n" diff --git a/panda/src/downloader/httpChannel.h b/panda/src/downloader/httpChannel.h index 94e4e4463d..b58b28271a 100644 --- a/panda/src/downloader/httpChannel.h +++ b/panda/src/downloader/httpChannel.h @@ -101,17 +101,22 @@ PUBLISHED: INLINE bool post_form(const URLSpec &url, const string &body); INLINE bool get_document(const URLSpec &url); + INLINE bool get_subdocument(const URLSpec &url, + size_t first_byte, size_t last_byte); INLINE bool get_header(const URLSpec &url); - INLINE void request_post_form(const URLSpec &url, const string &body); - INLINE void request_document(const URLSpec &url); - INLINE void request_header(const URLSpec &url); + INLINE void begin_post_form(const URLSpec &url, const string &body); + INLINE void begin_document(const URLSpec &url); + INLINE void begin_subdocument(const URLSpec &url, + size_t first_byte, size_t last_byte); + INLINE void begin_header(const URLSpec &url); bool run(); ISocketStream *read_body(); bool download_to_file(const Filename &filename); bool download_to_ram(Ramfile *ramfile); + INLINE size_t get_bytes_downloaded() const; INLINE bool is_download_complete() const; private: @@ -135,12 +140,14 @@ private: bool run_download_to_ram(); void begin_request(const string &method, const URLSpec &url, - const string &body, bool nonblocking); + const string &body, bool nonblocking, + size_t first_byte, size_t last_byte); bool http_getline(string &str); bool http_send(const string &str); bool parse_http_response(const string &line); bool parse_http_header(); + bool parse_content_range(const string &content_range); INLINE void check_socket(); bool verify_server(X509_NAME *subject) const; @@ -185,6 +192,8 @@ private: string _method; string _header; string _body; + size_t _first_byte; + size_t _last_byte; enum DownloadDest { DD_none, @@ -209,6 +218,7 @@ private: Headers _headers; size_t _file_size; + size_t _bytes_downloaded; // These members are used to maintain the current state while // communicating with the server. We need to store everything in