From 35348fd74a560257876c3f9023868b2dfa708d97 Mon Sep 17 00:00:00 2001 From: rdb Date: Wed, 22 Feb 2023 16:35:12 +0100 Subject: [PATCH] express: Do not rely on OpenSSL for MD5 hash support These hashes are used in various places in the Panda codebase (for integrity checks, not for crypto), so using an internal implementation allows retaining this functionality when building Panda without OpenSSL. Based on the following public domain implementation: https://github.com/B-Con/crypto-algorithms --- panda/src/downloader/patcher.cxx | 4 - panda/src/downloader/patcher.h | 4 - panda/src/express/hashVal.I | 24 +-- panda/src/express/hashVal.cxx | 250 +++++++++++++++++++++------ panda/src/express/hashVal.h | 8 +- panda/src/express/patchfile.cxx | 4 - panda/src/express/patchfile.h | 5 - panda/src/putil/bamCache.cxx | 18 +- panda/src/putil/load_prc_file.cxx | 5 - panda/src/putil/load_prc_file.h | 5 - pandatool/src/mayaprogs/mayapath.cxx | 31 +--- tests/express/test_hashval.py | 24 +++ 12 files changed, 245 insertions(+), 137 deletions(-) diff --git a/panda/src/downloader/patcher.cxx b/panda/src/downloader/patcher.cxx index c0b86f2c61..6ba4187f9f 100644 --- a/panda/src/downloader/patcher.cxx +++ b/panda/src/downloader/patcher.cxx @@ -13,8 +13,6 @@ #include "pandabase.h" -#ifdef HAVE_OPENSSL - #include "config_downloader.h" #include "patcher.h" #include "filename.h" @@ -71,5 +69,3 @@ int Patcher:: run() { return _patchfile->run(); } - -#endif // HAVE_OPENSSL diff --git a/panda/src/downloader/patcher.h b/panda/src/downloader/patcher.h index bc2d30e7bc..1261a8346c 100644 --- a/panda/src/downloader/patcher.h +++ b/panda/src/downloader/patcher.h @@ -16,8 +16,6 @@ #include "pandabase.h" -#ifdef HAVE_OPENSSL - #include "filename.h" #include "buffer.h" #include "patchfile.h" @@ -44,6 +42,4 @@ private: #include "patcher.I" -#endif // HAVE_OPENSSL - #endif diff --git a/panda/src/express/hashVal.I b/panda/src/express/hashVal.I index 9c67130f52..1b1cbe0634 100644 --- a/panda/src/express/hashVal.I +++ b/panda/src/express/hashVal.I @@ -164,11 +164,8 @@ read_stream(StreamReader &source) { _hv[3] = source.get_uint32(); } -#ifdef HAVE_OPENSSL /** - * Generates the hash value by hashing the indicated data. This method is - * only defined if we have the OpenSSL library (which provides md5 - * functionality) available. + * Generates the hash value by hashing the indicated data. */ INLINE void HashVal:: hash_ramfile(const Ramfile &ramfile) { @@ -176,9 +173,7 @@ hash_ramfile(const Ramfile &ramfile) { } /** - * Generates the hash value by hashing the indicated data. This method is - * only defined if we have the OpenSSL library (which provides md5 - * functionality) available. + * Generates the hash value by hashing the indicated data. */ INLINE void HashVal:: hash_string(const std::string &data) { @@ -186,15 +181,12 @@ hash_string(const std::string &data) { } /** - * Generates the hash value by hashing the indicated data. This method is - * only defined if we have the OpenSSL library (which provides md5 - * functionality) available. + * Generates the hash value by hashing the indicated data. */ INLINE void HashVal:: hash_bytes(const vector_uchar &data) { hash_buffer((const char *)&data[0], data.size()); } -#endif // HAVE_OPENSSL /** * Converts a single nibble to a hex digit. @@ -220,6 +212,16 @@ fromhex(char digit) { } } +/** + * Initializes the MD5 state. + */ +INLINE void HashVal:: +md5_init() { + _hv[0] = 0x67452301; + _hv[1] = 0xefcdab89; + _hv[2] = 0x98badcfe; + _hv[3] = 0x10325476; +} INLINE std::ostream &operator << (std::ostream &out, const HashVal &hv) { hv.output(out); diff --git a/panda/src/express/hashVal.cxx b/panda/src/express/hashVal.cxx index 479f91f752..ed1b60b530 100644 --- a/panda/src/express/hashVal.cxx +++ b/panda/src/express/hashVal.cxx @@ -15,18 +15,12 @@ #include "virtualFileSystem.h" #include -#ifdef HAVE_OPENSSL -#include "openSSLWrapper.h" // must be included before any other openssl. -#include -#endif // HAVE_OPENSSL - using std::istream; using std::istringstream; using std::ostream; using std::ostringstream; using std::string; - /** * Outputs the HashVal as a 32-digit hexadecimal number. */ @@ -169,11 +163,9 @@ set_from_bin(const vector_uchar &text) { return true; } -#ifdef HAVE_OPENSSL /** * Generates the hash value from the indicated file. Returns true on success, - * false if the file cannot be read. This method is only defined if we have - * the OpenSSL library (which provides md5 functionality) available. + * false if the file cannot be read. */ bool HashVal:: hash_file(const Filename &filename) { @@ -190,75 +182,75 @@ hash_file(const Filename &filename) { return result; } -#endif // HAVE_OPENSSL -#ifdef HAVE_OPENSSL /** * Generates the hash value from the indicated file. Returns true on success, - * false if the file cannot be read. This method is only defined if we have - * the OpenSSL library (which provides md5 functionality) available. + * false if the file cannot be read. */ bool HashVal:: hash_stream(istream &stream) { - unsigned char md[16]; + static const size_t buffer_size = 1024; // must be multiple of 64 + unsigned char buffer[buffer_size]; + unsigned char *ptr = buffer; + uint64_t total_count = 0; - MD5_CTX ctx; - MD5_Init(&ctx); - - static const int buffer_size = 1024; - char buffer[buffer_size]; + md5_init(); // Seek the stream to the beginning in case it wasn't there already. stream.seekg(0, std::ios::beg); - stream.read(buffer, buffer_size); + stream.read((char *)buffer, buffer_size); size_t count = stream.gcount(); while (count != 0) { - MD5_Update(&ctx, buffer, count); - stream.read(buffer, buffer_size); + total_count += (uint64_t)count; + + // Feed the data to the MD5 algorithm 64 bytes at a time. + while (count >= 64) { + md5_update(ptr); + ptr += 64; + count -= 64; + } + if (count > 0) { + break; + } + + stream.read((char *)buffer, buffer_size); + ptr = buffer; count = stream.gcount(); } + md5_final(ptr, count, total_count); + // Clear the fail bit so the caller can still read the stream (if it wants // to). stream.clear(); - MD5_Final(md, &ctx); - - // Store the individual bytes as big-endian ints, from historical - // convention. - _hv[0] = (md[0] << 24) | (md[1] << 16) | (md[2] << 8) | (md[3]); - _hv[1] = (md[4] << 24) | (md[5] << 16) | (md[6] << 8) | (md[7]); - _hv[2] = (md[8] << 24) | (md[9] << 16) | (md[10] << 8) | (md[11]); - _hv[3] = (md[12] << 24) | (md[13] << 16) | (md[14] << 8) | (md[15]); - return true; } -#endif // HAVE_OPENSSL - -#ifdef HAVE_OPENSSL /** - * Generates the hash value by hashing the indicated data. This method is - * only defined if we have the OpenSSL library (which provides md5 - * functionality) available. + * Generates the hash value by hashing the indicated data. */ void HashVal:: -hash_buffer(const char *buffer, int length) { - unsigned char md[16]; - MD5((const unsigned char *)buffer, length, md); +hash_buffer(const char *buffer, size_t length) { + md5_init(); - // Store the individual bytes as big-endian ints, from historical - // convention. - _hv[0] = (md[0] << 24) | (md[1] << 16) | (md[2] << 8) | (md[3]); - _hv[1] = (md[4] << 24) | (md[5] << 16) | (md[6] << 8) | (md[7]); - _hv[2] = (md[8] << 24) | (md[9] << 16) | (md[10] << 8) | (md[11]); - _hv[3] = (md[12] << 24) | (md[13] << 16) | (md[14] << 8) | (md[15]); + // Feed the data to the MD5 algorithm 64 bytes at a time. + const unsigned char *ptr = (const unsigned char *)buffer; + size_t remaining = length; + while (remaining >= 64) { + md5_update(ptr); + ptr += 64; + remaining -= 64; + } + + // md5_final needs a mutable buffer that is at least 64 bytes in size, since + // it needs to append extra padding data to the end. + unsigned char final_buffer[64]; + memcpy(final_buffer, ptr, remaining); + md5_final(final_buffer, remaining, length); } -#endif // HAVE_OPENSSL - - /** * Encodes the indicated unsigned int into an eight-digit hex string, stored * at the indicated buffer and the following 8 positions. @@ -294,3 +286,163 @@ decode_hex(const char *buffer, uint32_t &val) { (bytes[6] << 4) | (bytes[7])); } + +/** + * Core of the MD5 algorithm. Hashes a 64-byte block. This should be called + * repeatedly to add more data, followed with a single call to md5_final(). + * Taken from a public domain implementation written by Brad Conte. + */ +void HashVal:: +md5_update(const unsigned char *buffer) { +#define ROT(a, b) ((a << b) | (a >> (32 - b))) + +#define F(x, y, z) ((x & y) | (~x & z)) +#define G(x, y, z) ((x & z) | (y & ~z)) +#define H(x, y, z) (x ^ y ^ z) +#define I(x, y, z) (y ^ (x | ~z)) + +#define FF(a, b, c, d, m, s, t) { a += F(b, c, d) + m + t; a = b + ROT(a, s); } +#define GG(a, b, c, d, m, s, t) { a += G(b, c, d) + m + t; a = b + ROT(a, s); } +#define HH(a, b, c, d, m, s, t) { a += H(b, c, d) + m + t; a = b + ROT(a, s); } +#define II(a, b, c, d, m, s, t) { a += I(b, c, d) + m + t; a = b + ROT(a, s); } + + // MD5 specifies big endian byte order, so reverse all the bytes upon input, + // and re-reverse them on output. + uint32_t m[16]; + for (size_t i = 0, j = 0; i < 16; ++i, j += 4) { +#ifdef WORDS_BIGENDIAN + m[i] = (buffer[j] << 24u) + (buffer[j + 1] << 16u) + (buffer[j + 2] << 8u) + (buffer[j + 3]); +#else + m[i] = (buffer[j]) + (buffer[j + 1] << 8u) + (buffer[j + 2] << 16u) + (buffer[j + 3] << 24u); +#endif + } + + uint32_t a = _hv[0]; + uint32_t b = _hv[1]; + uint32_t c = _hv[2]; + uint32_t d = _hv[3]; + + FF(a, b, c, d, m[0], 7, 0xd76aa478); + FF(d, a, b, c, m[1], 12, 0xe8c7b756); + FF(c, d, a, b, m[2], 17, 0x242070db); + FF(b, c, d, a, m[3], 22, 0xc1bdceee); + FF(a, b, c, d, m[4], 7, 0xf57c0faf); + FF(d, a, b, c, m[5], 12, 0x4787c62a); + FF(c, d, a, b, m[6], 17, 0xa8304613); + FF(b, c, d, a, m[7], 22, 0xfd469501); + FF(a, b, c, d, m[8], 7, 0x698098d8); + FF(d, a, b, c, m[9], 12, 0x8b44f7af); + FF(c, d, a, b, m[10], 17, 0xffff5bb1); + FF(b, c, d, a, m[11], 22, 0x895cd7be); + FF(a, b, c, d, m[12], 7, 0x6b901122); + FF(d, a, b, c, m[13], 12, 0xfd987193); + FF(c, d, a, b, m[14], 17, 0xa679438e); + FF(b, c, d, a, m[15], 22, 0x49b40821); + + GG(a, b, c, d, m[1], 5, 0xf61e2562); + GG(d, a, b, c, m[6], 9, 0xc040b340); + GG(c, d, a, b, m[11], 14, 0x265e5a51); + GG(b, c, d, a, m[0], 20, 0xe9b6c7aa); + GG(a, b, c, d, m[5], 5, 0xd62f105d); + GG(d, a, b, c, m[10], 9, 0x02441453); + GG(c, d, a, b, m[15], 14, 0xd8a1e681); + GG(b, c, d, a, m[4], 20, 0xe7d3fbc8); + GG(a, b, c, d, m[9], 5, 0x21e1cde6); + GG(d, a, b, c, m[14], 9, 0xc33707d6); + GG(c, d, a, b, m[3], 14, 0xf4d50d87); + GG(b, c, d, a, m[8], 20, 0x455a14ed); + GG(a, b, c, d, m[13], 5, 0xa9e3e905); + GG(d, a, b, c, m[2], 9, 0xfcefa3f8); + GG(c, d, a, b, m[7], 14, 0x676f02d9); + GG(b, c, d, a, m[12], 20, 0x8d2a4c8a); + + HH(a, b, c, d, m[5], 4, 0xfffa3942); + HH(d, a, b, c, m[8], 11, 0x8771f681); + HH(c, d, a, b, m[11], 16, 0x6d9d6122); + HH(b, c, d, a, m[14], 23, 0xfde5380c); + HH(a, b, c, d, m[1], 4, 0xa4beea44); + HH(d, a, b, c, m[4], 11, 0x4bdecfa9); + HH(c, d, a, b, m[7], 16, 0xf6bb4b60); + HH(b, c, d, a, m[10], 23, 0xbebfbc70); + HH(a, b, c, d, m[13], 4, 0x289b7ec6); + HH(d, a, b, c, m[0], 11, 0xeaa127fa); + HH(c, d, a, b, m[3], 16, 0xd4ef3085); + HH(b, c, d, a, m[6], 23, 0x04881d05); + HH(a, b, c, d, m[9], 4, 0xd9d4d039); + HH(d, a, b, c, m[12], 11, 0xe6db99e5); + HH(c, d, a, b, m[15], 16, 0x1fa27cf8); + HH(b, c, d, a, m[2], 23, 0xc4ac5665); + + II(a, b, c, d, m[0], 6, 0xf4292244); + II(d, a, b, c, m[7], 10, 0x432aff97); + II(c, d, a, b, m[14], 15, 0xab9423a7); + II(b, c, d, a, m[5], 21, 0xfc93a039); + II(a, b, c, d, m[12], 6, 0x655b59c3); + II(d, a, b, c, m[3], 10, 0x8f0ccc92); + II(c, d, a, b, m[10], 15, 0xffeff47d); + II(b, c, d, a, m[1], 21, 0x85845dd1); + II(a, b, c, d, m[8], 6, 0x6fa87e4f); + II(d, a, b, c, m[15], 10, 0xfe2ce6e0); + II(c, d, a, b, m[6], 15, 0xa3014314); + II(b, c, d, a, m[13], 21, 0x4e0811a1); + II(a, b, c, d, m[4], 6, 0xf7537e82); + II(d, a, b, c, m[11], 10, 0xbd3af235); + II(c, d, a, b, m[2], 15, 0x2ad7d2bb); + II(b, c, d, a, m[9], 21, 0xeb86d391); + + _hv[0] += a; + _hv[1] += b; + _hv[2] += c; + _hv[3] += d; + +#undef ROT +#undef F +#undef G +#undef H +#undef I +#undef FF +#undef GG +#undef HH +#undef II +} + +/** + * Finalize the MD5 algorithm. The buffer parameter must contain the remaining + * bytes (counted by size, which must be < 64), and must be able to hold at + * least 64 bytes. The total_length parameter indicates the total number of + * bytes of the input string. + */ +void HashVal:: +md5_final(unsigned char *buffer, size_t size, uint64_t total_length) { + nassertv(size < 64); + + if (size < 56) { + buffer[size++] = 0x80; + memset(buffer + size, 0, 56 - size); + } else { + buffer[size++] = 0x80; + memset(buffer + size, 0, 64 - size); + md5_update(buffer); + memset(buffer, 0, 56); + } + + // Append to the padding the total message's length in bits and update. + uint64_t bitlen = (uint64_t)total_length << 3; + buffer[56] = (unsigned char)(bitlen); + buffer[57] = (unsigned char)(bitlen >> 8); + buffer[58] = (unsigned char)(bitlen >> 16); + buffer[59] = (unsigned char)(bitlen >> 24); + buffer[60] = (unsigned char)(bitlen >> 32); + buffer[61] = (unsigned char)(bitlen >> 40); + buffer[62] = (unsigned char)(bitlen >> 48); + buffer[63] = (unsigned char)(bitlen >> 56); + md5_update(buffer); + +#ifndef WORDS_BIGENDIAN + // Swap endianness of _hv. + for (uint32_t &v : _hv) { + v = ((v << 8) & 0xff00ff00) | ((v >> 8) & 0x00ff00ff); + v = (v << 16) | (v >> 16); + } +#endif +} diff --git a/panda/src/express/hashVal.h b/panda/src/express/hashVal.h index 85b0cb8c3d..0d2d8fc6b3 100644 --- a/panda/src/express/hashVal.h +++ b/panda/src/express/hashVal.h @@ -63,14 +63,12 @@ PUBLISHED: INLINE void write_stream(StreamWriter &destination) const; INLINE void read_stream(StreamReader &source); -#ifdef HAVE_OPENSSL bool hash_file(const Filename &filename); bool hash_stream(std::istream &stream); INLINE void hash_ramfile(const Ramfile &ramfile); INLINE void hash_string(const std::string &data); INLINE void hash_bytes(const vector_uchar &data); - void hash_buffer(const char *buffer, int length); -#endif // HAVE_OPENSSL + void hash_buffer(const char *buffer, size_t length); private: static void encode_hex(uint32_t val, char *buffer); @@ -78,6 +76,10 @@ private: INLINE static char tohex(unsigned int nibble); INLINE static unsigned int fromhex(char digit); + INLINE void md5_init(); + void md5_update(const unsigned char *buffer); + void md5_final(unsigned char *buffer, size_t size, uint64_t total_length); + uint32_t _hv[4]; }; diff --git a/panda/src/express/patchfile.cxx b/panda/src/express/patchfile.cxx index 748c9a76ba..845b0f7e65 100644 --- a/panda/src/express/patchfile.cxx +++ b/panda/src/express/patchfile.cxx @@ -13,8 +13,6 @@ #include "pandabase.h" -#ifdef HAVE_OPENSSL - #include "config_express.h" #include "error_utils.h" #include "patchfile.h" @@ -1350,5 +1348,3 @@ patch_subfile(ostream &write_stream, return true; } - -#endif // HAVE_OPENSSL diff --git a/panda/src/express/patchfile.h b/panda/src/express/patchfile.h index d6458fef58..c258e88d4f 100644 --- a/panda/src/express/patchfile.h +++ b/panda/src/express/patchfile.h @@ -16,8 +16,6 @@ #include "pandabase.h" -#ifdef HAVE_OPENSSL - #include "pnotify.h" #include "filename.h" #include "plist.h" @@ -31,7 +29,6 @@ #include - /** * */ @@ -170,6 +167,4 @@ private: #include "patchfile.I" -#endif // HAVE_OPENSSL - #endif diff --git a/panda/src/putil/bamCache.cxx b/panda/src/putil/bamCache.cxx index de189bf4a1..c6fd7d5e88 100644 --- a/panda/src/putil/bamCache.cxx +++ b/panda/src/putil/bamCache.cxx @@ -965,28 +965,12 @@ do_read_record(const Filename &cache_pathname, bool read_data) { */ string BamCache:: hash_filename(const string &filename) { -#ifdef HAVE_OPENSSL - // With OpenSSl, use the MD5 hash of the filename. + // Use the MD5 hash of the filename. HashVal hv; hv.hash_string(filename); ostringstream strm; hv.output_hex(strm); return strm.str(); - -#else // HAVE_OPENSSL - // Without OpenSSL, don't get fancy; just build a simple hash. - unsigned int hash = 0; - for (string::const_iterator si = filename.begin(); - si != filename.end(); - ++si) { - hash = (hash * 9109) + (unsigned int)(*si); - } - - ostringstream strm; - strm << std::hex << std::setw(8) << std::setfill('0') << hash; - return strm.str(); - -#endif // HAVE_OPENSSL } /** diff --git a/panda/src/putil/load_prc_file.cxx b/panda/src/putil/load_prc_file.cxx index f40296b403..e2755185bb 100644 --- a/panda/src/putil/load_prc_file.cxx +++ b/panda/src/putil/load_prc_file.cxx @@ -112,9 +112,6 @@ unload_prc_file(ConfigPage *page) { return cp_mgr->delete_explicit_page(page); } - -#ifdef HAVE_OPENSSL - /** * Fills HashVal with the hash from the current prc file state as reported by * ConfigVariableManager::write_prc_variables(). @@ -126,5 +123,3 @@ hash_prc_variables(HashVal &hash) { cv_mgr->write_prc_variables(strm); hash.hash_string(strm.str()); } - -#endif // HAVE_OPENSSL diff --git a/panda/src/putil/load_prc_file.h b/panda/src/putil/load_prc_file.h index cf29e6c8a3..5a67e4fd45 100644 --- a/panda/src/putil/load_prc_file.h +++ b/panda/src/putil/load_prc_file.h @@ -60,8 +60,6 @@ load_prc_file_data(const std::string &name, const std::string &data); EXPCL_PANDA_PUTIL bool unload_prc_file(ConfigPage *page); -#ifdef HAVE_OPENSSL - /** * Fills HashVal with the hash from the current prc file state as reported by * ConfigVariableManager::write_prc_variables(). @@ -69,9 +67,6 @@ unload_prc_file(ConfigPage *page); EXPCL_PANDA_PUTIL void hash_prc_variables(HashVal &hash); -#endif // HAVE_OPENSSL - - END_PUBLISH #endif diff --git a/pandatool/src/mayaprogs/mayapath.cxx b/pandatool/src/mayaprogs/mayapath.cxx index 7cf30fb179..4cbf19324f 100644 --- a/pandatool/src/mayaprogs/mayapath.cxx +++ b/pandatool/src/mayaprogs/mayapath.cxx @@ -279,9 +279,7 @@ main(int argc, char *argv[]) { Filename openmaya_standard = get_openmaya_filename(standard_maya_location); if (openmaya_given != openmaya_standard) { -#ifdef HAVE_OPENSSL - // If we have OpenSSL, we can use it to check the md5 hashes of the - // DLL. + // Check the md5 hashes of the DLL. HashVal hash_given, hash_standard; if (!hash_standard.hash_file(openmaya_standard)) { // Couldn't read the standard file, so use the given one. @@ -302,33 +300,6 @@ main(int argc, char *argv[]) { } } } -#else // HAVE_OPENSSL - // Without OpenSSL, just check the DLL filesize only. - off_t size_given, size_standard; - size_standard = openmaya_standard.get_file_size(); - if (size_standard == 0) { - // Couldn't read the standard file, so use the given one. - - } else { - size_given = openmaya_given.get_file_size(); - if (size_given == 0) { - // Couldn't even read the given file; use the standard one - // instead. - maya_location = standard_maya_location; - - } else { - if (size_standard != size_given) { - // No match; it must be the wrong version. - cerr << "$MAYA_LOCATION points to wrong version; using standard location instead.\n"; - maya_location = standard_maya_location; - - } else { - // The size matches; keep the given MAYA_LOCATION setting. - } - } - } - -#endif // HAVE_OPENSSL } } } diff --git a/tests/express/test_hashval.py b/tests/express/test_hashval.py index b2334e24d9..ec626060d0 100644 --- a/tests/express/test_hashval.py +++ b/tests/express/test_hashval.py @@ -1,5 +1,7 @@ from panda3d import core import random +import hashlib +import pytest def test_hashval_hex(): @@ -7,3 +9,25 @@ def test_hashval_hex(): val = core.HashVal() val.input_hex(core.StringStream(hex.encode('ascii'))) assert str(val) == hex.lower() + + +@pytest.mark.skipif('md5' not in hashlib.algorithms_available, + reason="MD5 algorithm not available in hashlib") +def test_hashval_md5(): + data = bytearray() + + for i in range(2500): + control = hashlib.md5(data).hexdigest() + + # Test hash_bytes + hv = core.HashVal() + hv.hash_bytes(bytes(data)) + assert hv.as_hex() == control + + # Test hash_stream + hv = core.HashVal() + result = hv.hash_stream(core.StringStream(data)) + assert result + assert hv.as_hex() == control + + data.append(random.randint(0, 255))