From b3b6c13d7159b2c421b67b94ee90b1100c6cb647 Mon Sep 17 00:00:00 2001 From: David Rose Date: Wed, 23 Sep 2009 02:45:14 +0000 Subject: [PATCH] HostInfo.asMirror --- direct/src/p3d/AppRunner.py | 4 +- direct/src/p3d/HostInfo.py | 94 +++++++++++++++---- direct/src/p3d/PackageInfo.py | 170 +++++++++++++++++++++++++--------- direct/src/p3d/PatchMaker.py | 23 +++-- 4 files changed, 218 insertions(+), 73 deletions(-) diff --git a/direct/src/p3d/AppRunner.py b/direct/src/p3d/AppRunner.py index 8962a204c1..7a0a237243 100644 --- a/direct/src/p3d/AppRunner.py +++ b/direct/src/p3d/AppRunner.py @@ -486,9 +486,7 @@ class AppRunner(DirectObject): host = self.getHost(hostUrl) - try: - host.readContentsFile() - except ValueError: + if not host.readContentsFile(): if not host.downloadContentsFile(self.http): print "Host %s cannot be downloaded, cannot preload %s." % (hostUrl, name) return diff --git a/direct/src/p3d/HostInfo.py b/direct/src/p3d/HostInfo.py index cd662754f1..b856e223ab 100644 --- a/direct/src/p3d/HostInfo.py +++ b/direct/src/p3d/HostInfo.py @@ -8,8 +8,25 @@ class HostInfo: Panda3D packages. It is the Python equivalent of the P3DHost class in the core API. """ - def __init__(self, hostUrl, appRunner): + def __init__(self, hostUrl, appRunner = None, hostDir = None, + asMirror = False): + + """ You must specify either an appRunner or a hostDir to the + HostInfo constructor. + + If you pass asMirror = True, it means that this HostInfo + object is to be used to populate a "mirror" folder, a + duplicate (or subset) of the contents hosted by a server. + This means when you use this HostInfo to download packages, it + will only download the compressed archive file and leave it + there. At the moment, mirror folders do not download old + patch files from the server. """ + + assert appRunner or hostDir + self.hostUrl = hostUrl + self.hostDir = hostDir + self.asMirror = asMirror # hostUrlPrefix is the host URL, but it is guaranteed to end # with a slash. @@ -39,7 +56,10 @@ class HostInfo: # will be filled in when the contents file is read. self.packages = {} - self.__determineHostDir(appRunner) + if appRunner: + self.__determineHostDir(appRunner) + + assert self.hostDir self.importsDir = Filename(self.hostDir, 'imports') def downloadContentsFile(self, http): @@ -76,32 +96,60 @@ class HostInfo: f.write(rf.getData()) f.close() - try: - self.readContentsFile() - except ValueError: + if not self.readContentsFile(): print "Failure reading %s" % (filename) return False return True + def redownloadContentsFile(self, http): + """ Downloads a new contents.xml file in case it has changed. + Returns true if the file has indeed changed, false if it has + not. """ + assert self.hasContentsFile + + url = self.hostUrlPrefix + 'contents.xml' + print "Redownloading %s" % (url) + + # Get the hash of the original file. + filename = Filename(self.hostDir, 'contents.xml') + hv1 = HashVal() + hv1.hashFile(filename) + + # Now download it again. + self.hasContentsFile = False + if not self.downloadContentsFile(http): + return False + + hv2 = HashVal() + hv2.hashFile(filename) + + if hv1 != hv2: + print "%s has changed." % (url) + return True + else: + print "%s has not changed." % (url) + return False + + def readContentsFile(self): - """ Reads the contents.xml file for this particular host. - Raises ValueError if the contents file is not already on disk - or is unreadable. """ + """ Reads the contents.xml file for this particular host, once + it has been downloaded. Returns true on success, false if the + contents file is not already on disk or is unreadable. """ if self.hasContentsFile: # No need to read it again. - return + return True filename = Filename(self.hostDir, 'contents.xml') doc = TiXmlDocument(filename.toOsSpecific()) if not doc.LoadFile(): - raise ValueError - + return False + xcontents = doc.FirstChildElement('contents') if not xcontents: - raise ValueError + return False # Look for our own entry in the hosts table. self.__findHostXml(xcontents) @@ -112,7 +160,12 @@ class HostInfo: name = xpackage.Attribute('name') platform = xpackage.Attribute('platform') version = xpackage.Attribute('version') - package = self.__makePackage(name, platform, version) + try: + solo = int(xpackage.Attribute('solo') or '') + except ValueError: + solo = False + + package = self.__makePackage(name, platform, version, solo) package.descFile = FileSpec() package.descFile.loadXml(xpackage) package.setupFilenames() @@ -127,6 +180,8 @@ class HostInfo: self.hasContentsFile = True + return True + def __findHostXml(self, xcontents): """ Looks for the or entry in the contents.xml that corresponds to the URL that we actually @@ -175,7 +230,7 @@ class HostInfo: self.altHosts[keyword] = url xalthost = xalthost.NextSiblingElement('alt_host') - def __makePackage(self, name, platform, version): + def __makePackage(self, name, platform, version, solo): """ Creates a new PackageInfo entry for the given name, version, and platform. If there is already a matching PackageInfo, returns it. """ @@ -188,7 +243,8 @@ class HostInfo: platforms = self.packages.setdefault((name, version), {}) package = platforms.get(platform, None) if not package: - package = PackageInfo(self, name, version, platform = platform) + package = PackageInfo(self, name, version, platform = platform, + solo = solo, asMirror = self.asMirror) platforms[platform] = package return package @@ -217,19 +273,19 @@ class HostInfo: return package - def getPackages(self, name, platform = None): + def getPackages(self, name = None, platform = None): """ Returns a list of PackageInfo objects that match the indicated name and/or platform, with no particular regards to - version. """ + version. If name is None, all packages are returned. """ assert self.hasContentsFile packages = [] for (pn, version), platforms in self.packages.items(): - if pn != name: + if name and pn != name: continue - package = self.getPackage(name, version, platform = platform) + package = self.getPackage(pn, version, platform = platform) if package: packages.append(package) diff --git a/direct/src/p3d/PackageInfo.py b/direct/src/p3d/PackageInfo.py index 1229b981bb..d0abb15619 100644 --- a/direct/src/p3d/PackageInfo.py +++ b/direct/src/p3d/PackageInfo.py @@ -19,6 +19,13 @@ class PackageInfo: unpackFactor = 0.01 patchFactor = 0.01 + class RestartDownload(Exception): + """ This exception is raised by __downloadFile() when the + toplevel contents.xml file has changed during the download, + and we have to restart from the beginning. """ + pass + + class InstallStep: """ This class is one step of the installPlan list; it represents a single atomic piece of the installation step, and @@ -41,16 +48,25 @@ class PackageInfo: return 1 return min(float(self.bytesDone) / float(self.bytesNeeded), 1) - def __init__(self, host, packageName, packageVersion, platform = None): + def __init__(self, host, packageName, packageVersion, platform = None, + solo = False, asMirror = False): self.host = host self.packageName = packageName self.packageVersion = packageVersion self.platform = platform + self.solo = solo + self.asMirror = asMirror self.packageDir = Filename(host.hostDir, self.packageName) if self.packageVersion: self.packageDir = Filename(self.packageDir, self.packageVersion) + if self.asMirror: + # The server directory contains the platform name, though + # the client directory doesn't. + if self.platform: + self.packageDir = Filename(self.packageDir, self.platform) + # These will be filled in by HostInfo when the package is read # from contents.xml. self.descFile = None @@ -126,16 +142,17 @@ class PackageInfo: if self.hasPackage: return True - if not self.hasDescFile: - filename = Filename(self.packageDir, self.descFileBasename) - if self.descFile.quickVerify(self.packageDir, pathname = filename): - self.readDescFile() - if self.hasDescFile: - # Successfully read. We don't need to call - # checkArchiveStatus again, since readDescFile() - # has just done it. - self.hasPackage = True - return True + try: + if not self.hasDescFile: + filename = Filename(self.packageDir, self.descFileBasename) + if self.descFile.quickVerify(self.packageDir, pathname = filename): + if self.__readDescFile(): + # Successfully read. We don't need to call + # checkArchiveStatus again, since readDescFile() + # has just done it. + return self.hasPackage + except self.RestartDownload: + return self.checkStatus() if self.hasDescFile: if self.__checkArchiveStatus(): @@ -156,33 +173,47 @@ class PackageInfo: return True self.http = http - if not self.__downloadFile( - None, self.descFile, - urlbase = self.descFile.filename, - filename = self.descFileBasename): - # Couldn't download the desc file. - return False - filename = Filename(self.packageDir, self.descFileBasename) - # Now that we've written the desc file, make it read-only. - os.chmod(filename.toOsSpecific(), 0444) + try: + if not self.__downloadFile( + None, self.descFile, + urlbase = self.descFile.filename, + filename = self.descFileBasename): + # Couldn't download the desc file. + return False - if not self.readDescFile(): - # Weird, it passed the hash check, but we still can't read - # it. - print "Failure reading %s" % (filename) - return False + filename = Filename(self.packageDir, self.descFileBasename) + # Now that we've written the desc file, make it read-only. + os.chmod(filename.toOsSpecific(), 0444) + + if not self.__readDescFile(): + # Weird, it passed the hash check, but we still can't read + # it. + print "Failure reading %s" % (filename) + return False + + except self.RestartDownload: + return self.downloadDescFile(http) return True - def readDescFile(self): - """ Reads the desc xml file for this particular package. - Returns true on success, false on failure. """ + def __readDescFile(self): + """ Reads the desc xml file for this particular package, + assuming it's been already downloaded and verified. Returns + true on success, false on failure. """ if self.hasDescFile: # No need to read it again. return True + if self.solo: + # If this is a "solo" package, we don't actually "read" + # the desc file; that's the entire contents of the + # package. + self.hasDescFile = True + self.hasPackage = True + return True + filename = Filename(self.packageDir, self.descFileBasename) doc = TiXmlDocument(filename.toOsSpecific()) @@ -230,15 +261,36 @@ class PackageInfo: # Now that we've read the desc file, go ahead and use it to # verify the download status. if self.__checkArchiveStatus(): - # It's all good. + # It's all fully downloaded, unpacked, and ready. self.hasPackage = True return True - # Now set up to download the update. + # Still have to download it. + self.__buildInstallPlan() + return True + + def __buildInstallPlan(self): + """ Sets up self.installPlans, a list of one or more "plans" + to download and install the package. """ + self.hasPackage = False - # Now determine what we will need to download, and build a - # plan (or two) to download it all. + if self.asMirror: + # If we're just downloading a mirror archive, we only need + # to get the compressed archive file. + + # Build a one-item install plan to download the compressed + # archive. + downloadSize = self.compressedArchive.size + func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec) + + step = self.InstallStep(func, downloadSize, self.downloadFactor) + installPlan = [step] + self.installPlans = [installPlan] + return + + # The normal download process. Determine what we will need to + # download, and build a plan (or two) to download it all. self.installPlans = None # We know we will at least need to unpackage the archive at @@ -254,7 +306,7 @@ class PackageInfo: self.uncompressedArchive.actualFile = None if self.uncompressedArchive.quickVerify(self.packageDir): self.installPlans = [planA] - return True + return # Maybe the compressed archive file is good. if self.compressedArchive.quickVerify(self.packageDir): @@ -262,7 +314,7 @@ class PackageInfo: step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor) planA = [step] + planA self.installPlans = [planA] - return True + return # Maybe we can download one or more patches. We'll come back # to that in a minute as plan A. For now, construct plan B, @@ -301,8 +353,6 @@ class PackageInfo: # plan B as the only plan. self.installPlans = [planB] - return True - def __scanDirectoryRecursively(self, dirname): """ Generates a list of Filename objects: all of the files (not directories) within and below the indicated dirname. """ @@ -332,11 +382,13 @@ class PackageInfo: # Get a list of all of the files in the directory, so we can # remove files that don't belong. - contents = self.__scanDirectoryRecursively(self.packageDir) - self.__removeFileFromList(contents, self.uncompressedArchive.filename) + contents = self.__scanDirectoryRecursively(self.packageDir) self.__removeFileFromList(contents, self.descFileBasename) - for file in self.extracts: - self.__removeFileFromList(contents, file.filename) + self.__removeFileFromList(contents, self.compressedArchive.filename) + if not self.asMirror: + self.__removeFileFromList(contents, self.uncompressedArchive.filename) + for file in self.extracts: + self.__removeFileFromList(contents, file.filename) # Now, any files that are still in the contents list don't # belong. It's important to remove these files before we @@ -347,6 +399,9 @@ class PackageInfo: print "Removing %s" % (filename) pathname = Filename(self.packageDir, filename) pathname.unlink() + + if self.asMirror: + return self.compressedArchive.quickVerify(self.packageDir) allExtractsOk = True if not self.uncompressedArchive.quickVerify(self.packageDir): @@ -354,6 +409,11 @@ class PackageInfo: allExtractsOk = False if allExtractsOk: + # OK, the uncompressed archive is good; that means there + # shouldn't be a compressed archive file here. + pathname = Filename(self.packageDir, self.compressedArchive.filename) + pathname.unlink() + for file in self.extracts: if not file.quickVerify(self.packageDir): #print "File is incorrect: %s" % (file.filename) @@ -378,7 +438,11 @@ class PackageInfo: def downloadPackage(self, http): """ Downloads the package file, synchronously, then uncompresses and unpacks it. Returns true on success, false - on failure. """ + on failure. + + This assumes that self.installPlans has already been filled + in, which will have been done by self.__readDescFile(). + """ assert self.hasDescFile @@ -388,9 +452,21 @@ class PackageInfo: # We should have an install plan by the time we get here. assert self.installPlans + installPlans = self.installPlans + self.installPlans = None + try: + return self.__followInstallPlans(installPlans, http) + + except self.RestartDownload: + if not self.downloadDescFile(http): + return False + return self.downloadPackage(http) + + + def __followInstallPlans(self, installPlans, http): self.http = http - for plan in self.installPlans: + for plan in installPlans: self.totalPlanSize = sum(map(lambda step: step.getEffort(), plan)) self.totalPlanCompleted = 0 self.downloadProgress = 0 @@ -502,12 +578,18 @@ class PackageInfo: continue if not fileSpec.fullVerify(self.packageDir, pathname = targetPathname): - print "After downloading, %s incorrect" % (url) + print "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename()) continue return True - # All mirrors failed. + # All mirrors failed. Maybe the original contents.xml file is + # stale. Try re-downloading it, in desperation. + if self.host.redownloadContentsFile(self.http): + raise self.RestartDownload + + # Nope, nothing's changed; the server (or the internet + # connection) must be just fubar. return False def __applyPatch(self, step, patchfile): diff --git a/direct/src/p3d/PatchMaker.py b/direct/src/p3d/PatchMaker.py index 8ed2c76095..b131df9d4b 100644 --- a/direct/src/p3d/PatchMaker.py +++ b/direct/src/p3d/PatchMaker.py @@ -247,7 +247,7 @@ class PatchMaker: """ This is a particular package. This contains all of the information needed to reconstruct the package's desc file. """ - def __init__(self, packageDesc, patchMaker, xpackage): + def __init__(self, packageDesc, patchMaker, xpackage = None): self.packageDir = Filename(patchMaker.installDir, packageDesc.getDirname()) self.packageDesc = packageDesc self.patchMaker = patchMaker @@ -256,6 +256,13 @@ class PatchMaker: self.currentPv = None self.basePv = None + self.packageName = None + self.platform = None + self.version = None + self.host = None + self.currentFile = None + self.baseFile = None + self.doc = None self.anyChanges = False self.patches = [] @@ -279,6 +286,7 @@ class PatchMaker: packageDescFullpath = Filename(self.patchMaker.installDir, self.packageDesc) self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific()) if not self.doc.LoadFile(): + print "Couldn't read %s" % (packageDescFullpath) return xpackage = self.doc.FirstChildElement('package') @@ -420,12 +428,13 @@ class PatchMaker: self.doc.SaveFile() - # Now that we've rewritten the xml file, we have to change - # the contents.xml file that references it to indicate the - # new file hash. - fileSpec = FileSpec() - fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc) - fileSpec.storeXml(self.contentsDocPackage) + if self.contentsDocPackage: + # Now that we've rewritten the xml file, we have to + # change the contents.xml file that references it to + # indicate the new file hash. + fileSpec = FileSpec() + fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc) + fileSpec.storeXml(self.contentsDocPackage) def __init__(self, installDir):