RomanNum3ral
/
open_toontown_panda3d
mirror of https://github.com/open-toontown/panda3d.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HostInfo.asMirror

This commit is contained in:
David Rose 2009-09-23 02:45:14 +00:00
parent 029512d577
commit b3b6c13d71
4 changed files with 218 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
direct/src/p3d/AppRunner.py
View File

@ -486,9 +486,7 @@ class AppRunner(DirectObject):
host = self.getHost(hostUrl)
try:
host.readContentsFile()
except ValueError:
if not host.readContentsFile():
if not host.downloadContentsFile(self.http):
print "Host %s cannot be downloaded, cannot preload %s." % (hostUrl, name)
return

94
direct/src/p3d/HostInfo.py
View File

@ -8,8 +8,25 @@ class HostInfo:
Panda3D packages. It is the Python equivalent of the P3DHost
class in the core API. """
def __init__(self, hostUrl, appRunner):
def __init__(self, hostUrl, appRunner = None, hostDir = None,
asMirror = False):
""" You must specify either an appRunner or a hostDir to the
HostInfo constructor.
If you pass asMirror = True, it means that this HostInfo
object is to be used to populate a "mirror" folder, a
duplicate (or subset) of the contents hosted by a server.
This means when you use this HostInfo to download packages, it
will only download the compressed archive file and leave it
there. At the moment, mirror folders do not download old
patch files from the server. """
assert appRunner or hostDir
self.hostUrl = hostUrl
self.hostDir = hostDir
self.asMirror = asMirror
# hostUrlPrefix is the host URL, but it is guaranteed to end
# with a slash.
@ -39,7 +56,10 @@ class HostInfo:
# will be filled in when the contents file is read.
self.packages = {}
self.__determineHostDir(appRunner)
if appRunner:
self.__determineHostDir(appRunner)
assert self.hostDir
self.importsDir = Filename(self.hostDir, 'imports')
def downloadContentsFile(self, http):
@ -76,32 +96,60 @@ class HostInfo:
f.write(rf.getData())
f.close()
try:
self.readContentsFile()
except ValueError:
if not self.readContentsFile():
print "Failure reading %s" % (filename)
return False
return True
def redownloadContentsFile(self, http):
""" Downloads a new contents.xml file in case it has changed.
Returns true if the file has indeed changed, false if it has
not. """
assert self.hasContentsFile
url = self.hostUrlPrefix + 'contents.xml'
print "Redownloading %s" % (url)
# Get the hash of the original file.
filename = Filename(self.hostDir, 'contents.xml')
hv1 = HashVal()
hv1.hashFile(filename)
# Now download it again.
self.hasContentsFile = False
if not self.downloadContentsFile(http):
return False
hv2 = HashVal()
hv2.hashFile(filename)
if hv1 != hv2:
print "%s has changed." % (url)
return True
else:
print "%s has not changed." % (url)
return False
def readContentsFile(self):
""" Reads the contents.xml file for this particular host.
Raises ValueError if the contents file is not already on disk
or is unreadable. """
""" Reads the contents.xml file for this particular host, once
it has been downloaded. Returns true on success, false if the
contents file is not already on disk or is unreadable. """
if self.hasContentsFile:
# No need to read it again.
return
return True
filename = Filename(self.hostDir, 'contents.xml')
doc = TiXmlDocument(filename.toOsSpecific())
if not doc.LoadFile():
raise ValueError
return False
xcontents = doc.FirstChildElement('contents')
if not xcontents:
raise ValueError
return False
# Look for our own entry in the hosts table.
self.__findHostXml(xcontents)
@ -112,7 +160,12 @@ class HostInfo:
name = xpackage.Attribute('name')
platform = xpackage.Attribute('platform')
version = xpackage.Attribute('version')
package = self.__makePackage(name, platform, version)
try:
solo = int(xpackage.Attribute('solo') or '')
except ValueError:
solo = False
package = self.__makePackage(name, platform, version, solo)
package.descFile = FileSpec()
package.descFile.loadXml(xpackage)
package.setupFilenames()
@ -127,6 +180,8 @@ class HostInfo:
self.hasContentsFile = True
return True
def __findHostXml(self, xcontents):
""" Looks for the <host> or <alt_host> entry in the
contents.xml that corresponds to the URL that we actually
@ -175,7 +230,7 @@ class HostInfo:
self.altHosts[keyword] = url
xalthost = xalthost.NextSiblingElement('alt_host')
def __makePackage(self, name, platform, version):
def __makePackage(self, name, platform, version, solo):
""" Creates a new PackageInfo entry for the given name,
version, and platform. If there is already a matching
PackageInfo, returns it. """
@ -188,7 +243,8 @@ class HostInfo:
platforms = self.packages.setdefault((name, version), {})
package = platforms.get(platform, None)
if not package:
package = PackageInfo(self, name, version, platform = platform)
package = PackageInfo(self, name, version, platform = platform,
solo = solo, asMirror = self.asMirror)
platforms[platform] = package
return package
@ -217,19 +273,19 @@ class HostInfo:
return package
def getPackages(self, name, platform = None):
def getPackages(self, name = None, platform = None):
""" Returns a list of PackageInfo objects that match the
indicated name and/or platform, with no particular regards to
version. """
version. If name is None, all packages are returned. """
assert self.hasContentsFile
packages = []
for (pn, version), platforms in self.packages.items():
if pn != name:
if name and pn != name:
continue
package = self.getPackage(name, version, platform = platform)
package = self.getPackage(pn, version, platform = platform)
if package:
packages.append(package)

170
direct/src/p3d/PackageInfo.py
View File

@ -19,6 +19,13 @@ class PackageInfo:
unpackFactor = 0.01
patchFactor = 0.01
class RestartDownload(Exception):
""" This exception is raised by __downloadFile() when the
toplevel contents.xml file has changed during the download,
and we have to restart from the beginning. """
pass
class InstallStep:
""" This class is one step of the installPlan list; it
represents a single atomic piece of the installation step, and
@ -41,16 +48,25 @@ class PackageInfo:
return 1
return min(float(self.bytesDone) / float(self.bytesNeeded), 1)
def __init__(self, host, packageName, packageVersion, platform = None):
def __init__(self, host, packageName, packageVersion, platform = None,
solo = False, asMirror = False):
self.host = host
self.packageName = packageName
self.packageVersion = packageVersion
self.platform = platform
self.solo = solo
self.asMirror = asMirror
self.packageDir = Filename(host.hostDir, self.packageName)
if self.packageVersion:
self.packageDir = Filename(self.packageDir, self.packageVersion)
if self.asMirror:
# The server directory contains the platform name, though
# the client directory doesn't.
if self.platform:
self.packageDir = Filename(self.packageDir, self.platform)
# These will be filled in by HostInfo when the package is read
# from contents.xml.
self.descFile = None
@ -126,16 +142,17 @@ class PackageInfo:
if self.hasPackage:
return True
if not self.hasDescFile:
filename = Filename(self.packageDir, self.descFileBasename)
if self.descFile.quickVerify(self.packageDir, pathname = filename):
self.readDescFile()
if self.hasDescFile:
# Successfully read. We don't need to call
# checkArchiveStatus again, since readDescFile()
# has just done it.
self.hasPackage = True
return True
try:
if not self.hasDescFile:
filename = Filename(self.packageDir, self.descFileBasename)
if self.descFile.quickVerify(self.packageDir, pathname = filename):
if self.__readDescFile():
# Successfully read. We don't need to call
# checkArchiveStatus again, since readDescFile()
# has just done it.
return self.hasPackage
except self.RestartDownload:
return self.checkStatus()
if self.hasDescFile:
if self.__checkArchiveStatus():
@ -156,33 +173,47 @@ class PackageInfo:
return True
self.http = http
if not self.__downloadFile(
None, self.descFile,
urlbase = self.descFile.filename,
filename = self.descFileBasename):
# Couldn't download the desc file.
return False
filename = Filename(self.packageDir, self.descFileBasename)
# Now that we've written the desc file, make it read-only.
os.chmod(filename.toOsSpecific(), 0444)
try:
if not self.__downloadFile(
None, self.descFile,
urlbase = self.descFile.filename,
filename = self.descFileBasename):
# Couldn't download the desc file.
return False
if not self.readDescFile():
# Weird, it passed the hash check, but we still can't read
# it.
print "Failure reading %s" % (filename)
return False
filename = Filename(self.packageDir, self.descFileBasename)
# Now that we've written the desc file, make it read-only.
os.chmod(filename.toOsSpecific(), 0444)
if not self.__readDescFile():
# Weird, it passed the hash check, but we still can't read
# it.
print "Failure reading %s" % (filename)
return False
except self.RestartDownload:
return self.downloadDescFile(http)
return True
def readDescFile(self):
""" Reads the desc xml file for this particular package.
Returns true on success, false on failure. """
def __readDescFile(self):
""" Reads the desc xml file for this particular package,
assuming it's been already downloaded and verified. Returns
true on success, false on failure. """
if self.hasDescFile:
# No need to read it again.
return True
if self.solo:
# If this is a "solo" package, we don't actually "read"
# the desc file; that's the entire contents of the
# package.
self.hasDescFile = True
self.hasPackage = True
return True
filename = Filename(self.packageDir, self.descFileBasename)
doc = TiXmlDocument(filename.toOsSpecific())
@ -230,15 +261,36 @@ class PackageInfo:
# Now that we've read the desc file, go ahead and use it to
# verify the download status.
if self.__checkArchiveStatus():
# It's all good.
# It's all fully downloaded, unpacked, and ready.
self.hasPackage = True
return True
# Now set up to download the update.
# Still have to download it.
self.__buildInstallPlan()
return True
def __buildInstallPlan(self):
""" Sets up self.installPlans, a list of one or more "plans"
to download and install the package. """
self.hasPackage = False
# Now determine what we will need to download, and build a
# plan (or two) to download it all.
if self.asMirror:
# If we're just downloading a mirror archive, we only need
# to get the compressed archive file.
# Build a one-item install plan to download the compressed
# archive.
downloadSize = self.compressedArchive.size
func = lambda step, fileSpec = self.compressedArchive: self.__downloadFile(step, fileSpec)
step = self.InstallStep(func, downloadSize, self.downloadFactor)
installPlan = [step]
self.installPlans = [installPlan]
return
# The normal download process. Determine what we will need to
# download, and build a plan (or two) to download it all.
self.installPlans = None
# We know we will at least need to unpackage the archive at
@ -254,7 +306,7 @@ class PackageInfo:
self.uncompressedArchive.actualFile = None
if self.uncompressedArchive.quickVerify(self.packageDir):
self.installPlans = [planA]
return True
return
# Maybe the compressed archive file is good.
if self.compressedArchive.quickVerify(self.packageDir):
@ -262,7 +314,7 @@ class PackageInfo:
step = self.InstallStep(self.__uncompressArchive, uncompressSize, self.uncompressFactor)
planA = [step] + planA
self.installPlans = [planA]
return True
return
# Maybe we can download one or more patches. We'll come back
# to that in a minute as plan A. For now, construct plan B,
@ -301,8 +353,6 @@ class PackageInfo:
# plan B as the only plan.
self.installPlans = [planB]
return True
def __scanDirectoryRecursively(self, dirname):
""" Generates a list of Filename objects: all of the files
(not directories) within and below the indicated dirname. """
@ -332,11 +382,13 @@ class PackageInfo:
# Get a list of all of the files in the directory, so we can
# remove files that don't belong.
contents = self.__scanDirectoryRecursively(self.packageDir)
self.__removeFileFromList(contents, self.uncompressedArchive.filename)
contents = self.__scanDirectoryRecursively(self.packageDir)
self.__removeFileFromList(contents, self.descFileBasename)
for file in self.extracts:
self.__removeFileFromList(contents, file.filename)
self.__removeFileFromList(contents, self.compressedArchive.filename)
if not self.asMirror:
self.__removeFileFromList(contents, self.uncompressedArchive.filename)
for file in self.extracts:
self.__removeFileFromList(contents, file.filename)
# Now, any files that are still in the contents list don't
# belong. It's important to remove these files before we
@ -347,6 +399,9 @@ class PackageInfo:
print "Removing %s" % (filename)
pathname = Filename(self.packageDir, filename)
pathname.unlink()
if self.asMirror:
return self.compressedArchive.quickVerify(self.packageDir)
allExtractsOk = True
if not self.uncompressedArchive.quickVerify(self.packageDir):
@ -354,6 +409,11 @@ class PackageInfo:
allExtractsOk = False
if allExtractsOk:
# OK, the uncompressed archive is good; that means there
# shouldn't be a compressed archive file here.
pathname = Filename(self.packageDir, self.compressedArchive.filename)
pathname.unlink()
for file in self.extracts:
if not file.quickVerify(self.packageDir):
#print "File is incorrect: %s" % (file.filename)
@ -378,7 +438,11 @@ class PackageInfo:
def downloadPackage(self, http):
""" Downloads the package file, synchronously, then
uncompresses and unpacks it. Returns true on success, false
on failure. """
on failure.
This assumes that self.installPlans has already been filled
in, which will have been done by self.__readDescFile().
"""
assert self.hasDescFile
@ -388,9 +452,21 @@ class PackageInfo:
# We should have an install plan by the time we get here.
assert self.installPlans
installPlans = self.installPlans
self.installPlans = None
try:
return self.__followInstallPlans(installPlans, http)
except self.RestartDownload:
if not self.downloadDescFile(http):
return False
return self.downloadPackage(http)
def __followInstallPlans(self, installPlans, http):
self.http = http
for plan in self.installPlans:
for plan in installPlans:
self.totalPlanSize = sum(map(lambda step: step.getEffort(), plan))
self.totalPlanCompleted = 0
self.downloadProgress = 0
@ -502,12 +578,18 @@ class PackageInfo:
continue
if not fileSpec.fullVerify(self.packageDir, pathname = targetPathname):
print "After downloading, %s incorrect" % (url)
print "After downloading, %s incorrect" % (Filename(fileSpec.filename).getBasename())
continue
return True
# All mirrors failed.
# All mirrors failed. Maybe the original contents.xml file is
# stale. Try re-downloading it, in desperation.
if self.host.redownloadContentsFile(self.http):
raise self.RestartDownload
# Nope, nothing's changed; the server (or the internet
# connection) must be just fubar.
return False
def __applyPatch(self, step, patchfile):

23
direct/src/p3d/PatchMaker.py
View File

@ -247,7 +247,7 @@ class PatchMaker:
""" This is a particular package. This contains all of the
information needed to reconstruct the package's desc file. """
def __init__(self, packageDesc, patchMaker, xpackage):
def __init__(self, packageDesc, patchMaker, xpackage = None):
self.packageDir = Filename(patchMaker.installDir, packageDesc.getDirname())
self.packageDesc = packageDesc
self.patchMaker = patchMaker
@ -256,6 +256,13 @@ class PatchMaker:
self.currentPv = None
self.basePv = None
self.packageName = None
self.platform = None
self.version = None
self.host = None
self.currentFile = None
self.baseFile = None
self.doc = None
self.anyChanges = False
self.patches = []
@ -279,6 +286,7 @@ class PatchMaker:
packageDescFullpath = Filename(self.patchMaker.installDir, self.packageDesc)
self.doc = TiXmlDocument(packageDescFullpath.toOsSpecific())
if not self.doc.LoadFile():
print "Couldn't read %s" % (packageDescFullpath)
return
xpackage = self.doc.FirstChildElement('package')
@ -420,12 +428,13 @@ class PatchMaker:
self.doc.SaveFile()
# Now that we've rewritten the xml file, we have to change
# the contents.xml file that references it to indicate the
# new file hash.
fileSpec = FileSpec()
fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
fileSpec.storeXml(self.contentsDocPackage)
if self.contentsDocPackage:
# Now that we've rewritten the xml file, we have to
# change the contents.xml file that references it to
# indicate the new file hash.
fileSpec = FileSpec()
fileSpec.fromFile(self.patchMaker.installDir, self.packageDesc)
fileSpec.storeXml(self.contentsDocPackage)
def __init__(self, installDir):