From ea1d78b85e49b9b0681d27ed95f0fb13c90e2385 Mon Sep 17 00:00:00 2001 From: David Rose Date: Tue, 19 Apr 2011 23:26:56 +0000 Subject: [PATCH] extend pmerge.py to validate and/or correct hashes --- direct/src/p3d/FileSpec.py | 46 ++++++++++++++++++++++--- direct/src/p3d/PackageMerger.py | 61 ++++++++++++++++++++++++++++++++- direct/src/p3d/pmerge.py | 16 ++++++--- 3 files changed, 112 insertions(+), 11 deletions(-) diff --git a/direct/src/p3d/FileSpec.py b/direct/src/p3d/FileSpec.py index d624cc10bc..cdc1f3bd4c 100644 --- a/direct/src/p3d/FileSpec.py +++ b/direct/src/p3d/FileSpec.py @@ -1,4 +1,5 @@ import os +import time from pandac.PandaModules import Filename, HashVal, VirtualFileSystem class FileSpec: @@ -88,13 +89,18 @@ class FileSpec: xelement.SetAttribute('hash', self.hash) def quickVerify(self, packageDir = None, pathname = None, - notify = None): + notify = None, correctSelf = False): """ Performs a quick test to ensure the file has not been modified. This test is vulnerable to people maliciously attempting to fool the program (by setting datestamps etc.). - Returns true if it is intact, false if it needs to be - redownloaded. """ + if correctSelf is True, then any discrepency is corrected by + updating the appropriate fields internally, making the + assumption that the file on disk is the authoritative version. + + Returns true if it is intact, false if it is incorrect. If + correctSelf is true, raises OSError if the self-update is + impossible (for instance, because the file does not exist).""" if not pathname: pathname = Filename(packageDir, self.filename) @@ -104,12 +110,16 @@ class FileSpec: # If the file is missing, the file fails. if notify: notify.debug("file not found: %s" % (pathname)) + if correctSelf: + raise return False if st.st_size != self.size: # If the size is wrong, the file fails. if notify: notify.debug("size wrong: %s" % (pathname)) + if correctSelf: + self.__correctHash(packageDir, pathname, st, notify) return False if st.st_mtime == self.timestamp: @@ -129,6 +139,8 @@ class FileSpec: if notify: notify.debug("hash check wrong: %s" % (pathname)) notify.debug(" found %s, expected %s" % (self.actualFile.hash, self.hash)) + if correctSelf: + self.__correctHash(packageDir, pathname, st, notify) return False if notify: @@ -137,7 +149,12 @@ class FileSpec: # The hash is OK after all. Change the file's timestamp back # to what we expect it to be, so we can quick-verify it # successfully next time. - self.__updateTimestamp(pathname, st) + if correctSelf: + # Or update our own timestamp. + self.__correctTimestamp(pathname, st, notify) + return False + else: + self.__updateTimestamp(pathname, st) return True @@ -194,6 +211,14 @@ class FileSpec: except OSError: pass + def __correctTimestamp(self, pathname, st, notify): + """ Corrects the internal timestamp to match the one on + disk. """ + if notify: + notify.info("Correcting timestamp of %s to %d (%s)" % ( + self.filename, st.st_mtime, time.asctime(time.localtime(st.st_mtime)))) + self.timestamp = st.st_mtime + def checkHash(self, packageDir, pathname, st): """ Returns true if the file has the expected md5 hash, false otherwise. As a side effect, stores a FileSpec corresponding @@ -205,4 +230,15 @@ class FileSpec: self.actualFile = fileSpec return (fileSpec.hash == self.hash) - + + def __correctHash(self, packageDir, pathname, st, notify): + """ Corrects the internal hash to match the one on disk. """ + if not self.actualFile: + self.checkHash(packageDir, pathname, st) + + if notify: + notify.info("Correcting hash %s to %s" % ( + self.filename, self.actualFile.hash)) + self.hash = self.actualFile.hash + self.size = self.actualFile.size + self.timestamp = self.actualFile.timestamp diff --git a/direct/src/p3d/PackageMerger.py b/direct/src/p3d/PackageMerger.py index b15dbadcfa..ac0c3f1b2f 100644 --- a/direct/src/p3d/PackageMerger.py +++ b/direct/src/p3d/PackageMerger.py @@ -1,5 +1,6 @@ from direct.p3d.FileSpec import FileSpec from direct.p3d.SeqValue import SeqValue +from direct.directnotify.DirectNotifyGlobal import * from pandac.PandaModules import * import copy import shutil @@ -15,6 +16,8 @@ class PackageMerger: hosts are in sync, so that the file across all builds with the most recent timestamp (indicated in the contents.xml file) is always the most current version of the file. """ + + notify = directNotify.newCategory("PackageMerger") class PackageEntry: """ This corresponds to a entry in the contents.xml @@ -42,6 +45,10 @@ class PackageMerger: self.descFile = FileSpec() self.descFile.loadXml(xpackage) + self.validatePackageContents() + + self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True) + self.packageSeq = SeqValue() self.packageSeq.loadXml(xpackage, 'seq') self.packageSetVer = SeqValue() @@ -52,6 +59,7 @@ class PackageMerger: if ximport: self.importDescFile = FileSpec() self.importDescFile.loadXml(ximport) + self.importDescFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True) def makeXml(self): """ Returns a new TiXmlElement. """ @@ -75,6 +83,50 @@ class PackageMerger: return xpackage + def validatePackageContents(self): + """ Validates the contents of the package directory itself + against the expected hashes and timestamps. Updates + hashes and timestamps where needed. """ + + if self.solo: + return + + needsChange = False + packageDescFullpath = Filename(self.sourceDir, self.descFile.filename) + packageDir = Filename(packageDescFullpath.getDirname()) + doc = TiXmlDocument(packageDescFullpath.toOsSpecific()) + if not doc.LoadFile(): + message = "Could not read XML file: %s" % (self.descFile.filename) + raise OSError, message + + xpackage = doc.FirstChildElement('package') + if not xpackage: + message = "No package definition: %s" % (self.descFile.filename) + raise OSError, message + + xcompressed = xpackage.FirstChildElement('compressed_archive') + if xcompressed: + spec = FileSpec() + spec.loadXml(xcompressed) + if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True): + spec.storeXml(xcompressed) + needsChange = True + + xpatch = xpackage.FirstChildElement('patch') + while xpatch: + spec = FileSpec() + spec.loadXml(xpatch) + if not spec.quickVerify(packageDir = packageDir, notify = PackageMerger.notify, correctSelf = True): + spec.storeXml(xpatch) + needsChange = True + + xpatch = xpatch.NextSiblingElement('patch') + + if needsChange: + PackageMerger.notify.info("Rewriting %s" % (self.descFile.filename)) + doc.SaveFile() + self.descFile.quickVerify(packageDir = self.sourceDir, notify = PackageMerger.notify, correctSelf = True) + # PackageMerger constructor def __init__(self, installDir): self.installDir = installDir @@ -161,7 +213,7 @@ class PackageMerger: there. """ dirname = Filename(pe.descFile.filename).getDirname() - print "copying %s" % (dirname) + self.notify.info("copying %s" % (dirname)) sourceDirname = Filename(pe.sourceDir, dirname) targetDirname = Filename(self.installDir, dirname) @@ -208,6 +260,13 @@ class PackageMerger: # Copying a regular file. sourceFilename.copyTo(targetFilename) + # Also try to copy the timestamp, but don't fuss too much + # if it doesn't work. + try: + st = os.stat(sourceFilename.toOsSpecific()) + os.utime(targetFilename.toOsSpecific(), (st.st_atime, st.st_mtime)) + except OSError: + pass def merge(self, sourceDir): """ Adds the contents of the indicated source directory into diff --git a/direct/src/p3d/pmerge.py b/direct/src/p3d/pmerge.py index af06ca0537..092cee0223 100755 --- a/direct/src/p3d/pmerge.py +++ b/direct/src/p3d/pmerge.py @@ -4,7 +4,9 @@ usageText = """ This script can be used to merge together the contents of two or more separately-built stage directories, built independently via ppackage, -or via Packager.py. +or via Packager.py. This script also verifies the hash, file size, +and timestamp values in the stage directory as it runs, so it can be +run on a single standalone directory just to perform this validation. This script is actually a wrapper around Panda's PackageMerger.py. @@ -24,7 +26,9 @@ Options: -i install_dir The full path to the final install directory. This may also contain some pre-existing contents; if so, it is merged with all - of the input directories as well. + of the input directories as well. The contents of this directory + are checked for self-consistency with regards to hashes and + timestamps. -h Display this help @@ -62,9 +66,11 @@ inputDirs = [] for arg in args: inputDirs.append(Filename.fromOsSpecific(arg)) -if not inputDirs: - print "no input directories specified." - sys.exit(1) +# It's now legal to have no input files if you only want to verify +# timestamps and hashes. +## if not inputDirs: +## print "no input directories specified." +## sys.exit(1) try: pm = PackageMerger.PackageMerger(installDir)