From 8f1e2f1945d9ae366b954d588255fa83eb638272 Mon Sep 17 00:00:00 2001 From: rdb Date: Sun, 14 Feb 2021 17:29:45 +0100 Subject: [PATCH] downloader: Support gzip and deflate Content-Encoding values --- panda/src/downloader/httpChannel.cxx | 67 +++++++++++++++++++++++----- panda/src/downloader/httpChannel.h | 5 ++- panda/src/downloader/httpClient.cxx | 25 +---------- 3 files changed, 60 insertions(+), 37 deletions(-) diff --git a/panda/src/downloader/httpChannel.cxx b/panda/src/downloader/httpChannel.cxx index f066b24943..c5d5b731b9 100644 --- a/panda/src/downloader/httpChannel.cxx +++ b/panda/src/downloader/httpChannel.cxx @@ -22,6 +22,11 @@ #include "virtualFileMountHTTP.h" #include "ramfile.h" #include "globPattern.h" +#include "string_utils.h" + +#ifdef HAVE_ZLIB +#include "zStream.h" +#endif #include @@ -111,6 +116,7 @@ HTTPChannel(HTTPClient *client) : _done_state = S_new; _started_download = false; _sent_so_far = 0; + _body_socket_stream = nullptr; _body_stream = nullptr; _owns_body_stream = false; _sbio = nullptr; @@ -550,7 +556,7 @@ run() { * The user is responsible for passing the returned istream to * close_read_body() later. */ -ISocketStream *HTTPChannel:: +std::istream *HTTPChannel:: open_read_body() { reset_body_stream(); @@ -560,14 +566,14 @@ open_read_body() { string transfer_coding = downcase(get_header_value("Transfer-Encoding")); - ISocketStream *result; + std::istream *result; if (transfer_coding == "chunked") { // "chunked" transfer encoding. This means we will have to decode the // length of the file as we read it in chunks. The IChunkedStream does // this. _state = S_reading_body; _read_index++; - result = new IChunkedStream(_source, this); + _body_socket_stream = new IChunkedStream(_source, this); } else { // If the transfer encoding is anything else, assume "identity". This is @@ -576,11 +582,39 @@ open_read_body() { // file otherwise. _state = S_reading_body; _read_index++; - result = new IIdentityStream(_source, this, _got_file_size, _file_size); + _body_socket_stream = new IIdentityStream(_source, this, _got_file_size, _file_size); + } + result = _body_socket_stream; + + string content_encoding = trim(get_header_value("Content-Encoding")); + if (!content_encoding.empty()) { + vector_string content_encodings; + tokenize(downcase(content_encoding), content_encodings, ","); + for (const string &encoding : content_encodings) { + string trimmed = trim(encoding); + if (trimmed == "identity") { + continue; + } +#ifdef HAVE_ZLIB + else if (trimmed == "gzip" || trimmed == "deflate" || trimmed == "x-gzip") { + // "deflate" actually includes zlib header, which is accepted as well + result = new IDecompressStream(result, true, -1, true); + } +#endif + else { + downloader_cat.error() + << "Content-Encoding not supported: " << trimmed << "\n"; + delete result; + _body_socket_stream = nullptr; + _body_stream = nullptr; + _owns_body_stream = false; + return nullptr; + } + } } - result->_channel = this; _body_stream = result; + _body_socket_stream->_channel = this; _owns_body_stream = false; return result; @@ -804,9 +838,9 @@ downcase(const string &s) { */ void HTTPChannel:: body_stream_destructs(ISocketStream *stream) { - if (stream == _body_stream) { + if (stream == _body_socket_stream) { if (_state == S_reading_body) { - switch (_body_stream->get_read_state()) { + switch (_body_socket_stream->get_read_state()) { case ISocketStream::RS_complete: finished_body(false); break; @@ -820,6 +854,8 @@ body_stream_destructs(ISocketStream *stream) { break; } } + + _body_socket_stream = nullptr; _body_stream = nullptr; _owns_body_stream = false; } @@ -2160,7 +2196,7 @@ run_reading_body() { std::getline(*_body_stream, line); } - if (!_body_stream->is_closed()) { + if (!_body_socket_stream->is_closed()) { // There's more to come later. return true; } @@ -2281,7 +2317,7 @@ run_download_to_file() { _download_to_stream->flush(); - if (_body_stream->is_closed()) { + if (_body_socket_stream->is_closed()) { // Done. reset_body_stream(); close_download_stream(); @@ -2331,7 +2367,7 @@ run_download_to_ram() { count = _body_stream->gcount(); } - if (_body_stream->is_closed()) { + if (_body_socket_stream->is_closed()) { // Done. reset_body_stream(); close_download_stream(); @@ -2392,7 +2428,7 @@ run_download_to_stream() { _download_to_stream->flush(); - if (_body_stream->is_closed()) { + if (_body_socket_stream->is_closed()) { // Done. reset_body_stream(); close_download_stream(); @@ -3635,6 +3671,14 @@ make_header() { << "Content-Length: " << _body.length() << "\r\n"; } +#ifdef HAVE_ZLIB + stream + << "Accept-Encoding: gzip, deflate, identity\r\n"; +#else + stream + << "Accept-Encoding: identity\r\n"; +#endif + _header = stream.str(); } @@ -3810,6 +3854,7 @@ reset_body_stream() { nassertv(_body_stream == nullptr && !_owns_body_stream); } } else { + _body_socket_stream = nullptr; _body_stream = nullptr; } } diff --git a/panda/src/downloader/httpChannel.h b/panda/src/downloader/httpChannel.h index 41d4edd121..f204bd40c3 100644 --- a/panda/src/downloader/httpChannel.h +++ b/panda/src/downloader/httpChannel.h @@ -182,7 +182,7 @@ PUBLISHED: bool run(); INLINE void begin_connect_to(const DocumentSpec &url); - ISocketStream *open_read_body(); + std::istream *open_read_body(); void close_read_body(std::istream *stream) const; BLOCKING bool download_to_file(const Filename &filename, bool subdocument_resumes = true); @@ -416,7 +416,8 @@ private: size_t _sent_so_far; std::string _current_field_name; std::string _current_field_value; - ISocketStream *_body_stream; + ISocketStream *_body_socket_stream; + std::istream *_body_stream; bool _owns_body_stream; BIO *_sbio; std::string _cipher_list; diff --git a/panda/src/downloader/httpClient.cxx b/panda/src/downloader/httpClient.cxx index c451f13084..ddb8af647c 100644 --- a/panda/src/downloader/httpClient.cxx +++ b/panda/src/downloader/httpClient.cxx @@ -21,6 +21,7 @@ #include "httpBasicAuthorization.h" #include "httpDigestAuthorization.h" #include "globPattern.h" +#include "string_utils.h" #ifdef HAVE_OPENSSL @@ -48,30 +49,6 @@ trim_blanks(const string &str) { return str.substr(start, end - start); } -/** - * Chops the source string up into pieces delimited by any of the characters - * specified in delimiters. Repeated delimiter characters represent zero- - * length tokens. - * - * It is the user's responsibility to ensure the output vector is cleared - * before calling this function; the results will simply be appended to the - * end of the vector. - */ -static void -tokenize(const string &str, vector_string &words, const string &delimiters) { - size_t p = 0; - while (p < str.length()) { - size_t q = str.find_first_of(delimiters, p); - if (q == string::npos) { - words.push_back(str.substr(p)); - return; - } - words.push_back(str.substr(p, q - p)); - p = q + 1; - } - words.push_back(string()); -} - #ifndef NDEBUG /** * This method is attached as a callback for SSL messages only when debug