diff --git a/direct/src/p3d/AppRunner.py b/direct/src/p3d/AppRunner.py index 736b818901..e44a0195cd 100644 --- a/direct/src/p3d/AppRunner.py +++ b/direct/src/p3d/AppRunner.py @@ -12,6 +12,7 @@ runp3d.py for a command-line tool to invoke this module. import sys import os import types +import shutil import __builtin__ if 'VFSImporter' in sys.modules: @@ -33,7 +34,7 @@ else: from direct.showbase import VFSImporter from direct.showbase.DirectObject import DirectObject -from pandac.PandaModules import VirtualFileSystem, Filename, Multifile, loadPrcFileData, unloadPrcFile, getModelPath, Thread, WindowProperties, ExecutionEnvironment, PandaSystem, Notify, StreamWriter, ConfigVariableString, initAppForGui +from pandac.PandaModules import VirtualFileSystem, Filename, Multifile, loadPrcFileData, unloadPrcFile, getModelPath, Thread, WindowProperties, ExecutionEnvironment, PandaSystem, Notify, StreamWriter, ConfigVariableString, initAppForGui, TiXmlDocument from pandac import PandaModules from direct.stdpy import file, glob from direct.task.TaskManagerGlobal import taskMgr @@ -41,6 +42,9 @@ from direct.showbase.MessengerGlobal import messenger from direct.showbase import AppRunnerGlobal from direct.directnotify.DirectNotifyGlobal import directNotify from direct.p3d.HostInfo import HostInfo +from direct.p3d.ScanDirectoryNode import ScanDirectoryNode +from direct.p3d.InstalledHostData import InstalledHostData +from direct.p3d.InstalledPackageData import InstalledPackageData # These imports are read by the C++ wrapper in p3dPythonRun.cxx. from direct.p3d.JavaScript import UndefinedObject, Undefined, ConcreteStruct, BrowserObject @@ -66,6 +70,11 @@ class AppRunner(DirectObject): development purposes. """ notify = directNotify.newCategory("AppRunner") + + ConfigBasename = 'config.xml' + + # Default values for parameters that are absent from the config file: + maxDiskUsage = 1073741824 # 1 GB def __init__(self): DirectObject.__init__(self) @@ -344,6 +353,63 @@ class AppRunner(DirectObject): self.hosts[hostUrl] = host return host + def getHostWithDir(self, hostDir): + """ Returns the HostInfo object that corresponds to the + indicated on-disk host directory. This would be used when + reading a host directory from disk, instead of downloading it + from a server. Supply the full path to the host directory, as + a Filename. Returns None if the contents.xml in the indicated + host directory cannot be read or doesn't seem consistent. """ + + host = HostInfo(None, hostDir = hostDir, appRunner = self) + if not host.readContentsFile(): + # Couldn't read the contents.xml file + return None + + if not host.hostUrl: + # The contents.xml file there didn't seem to indicate the + # same host directory. + return None + + host2 = self.hosts.get(host.hostUrl) + if host2 is None: + # No such host already; store this one. + self.hosts[host.hostUrl] = host + return host + + if host2.hostDir != host.hostDir: + # Hmm, we already have that host somewhere else. + return None + + # We already have that host, and it's consistent. + return host2 + + def deletePackages(self, packages): + """ Removes all of the indicated packages from the disk, + uninstalling them and deleting all of their files. The + packages parameter must be a list of one or more PackageInfo + objects, for instance as returned by getHost().getPackage(). + Returns the list of packages that were NOT found. """ + + for hostUrl, host in self.hosts.items(): + packages = host.deletePackages(packages) + + if not host.packages: + # If that's all of the packages for this host, delete + # the host directory too. + del self.hosts[hostUrl] + self.__deleteHostFiles(host) + + return packages + + def __deleteHostFiles(self, host): + """ Called by deletePackages(), this removes all the files for + the indicated host (for which we have presumably already + removed all of the packages). """ + + self.notify.info("Deleting host %s: %s" % (host.hostUrl, host.hostDir)) + shutil.rmtree(host.hostDir.toOsSpecific(), True) + def freshenFile(self, host, fileSpec, localPathname): """ Ensures that the localPathname is the most current version of the file defined by fileSpec, as offered by host. If not, @@ -392,6 +458,114 @@ class AppRunner(DirectObject): return True + def scanInstalledPackages(self): + """ Scans the hosts and packages already installed locally on + the system. Returns a list of InstalledHostData objects, each + of which contains a list of InstalledPackageData objects. """ + + result = [] + hostsFilename = Filename(self.rootDir, 'hosts') + hostsDir = ScanDirectoryNode(hostsFilename) + for dirnode in hostsDir.nested: + host = self.getHostWithDir(dirnode.pathname) + hostData = InstalledHostData(host, dirnode) + + if host: + for package in host.getAllPackages(): + packageDir = package.getPackageDir() + if not packageDir.exists(): + continue + + subdir = dirnode.extractSubdir(packageDir) + if not subdir: + # This package, while defined by the host, isn't installed + # locally; ignore it. + continue + + packageData = InstalledPackageData(package, subdir) + hostData.packages.append(packageData) + + # Now that we've examined all of the packages for the host, + # anything left over is junk. + for subdir in dirnode.nested: + packageData = InstalledPackageData(None, subdir) + hostData.packages.append(packageData) + + result.append(hostData) + + return result + + def readConfigXml(self): + """ Reads the config.xml file that may be present in the root + directory. """ + + if not hasattr(PandaModules, 'TiXmlDocument'): + return + + filename = Filename(self.rootDir, self.ConfigBasename) + doc = TiXmlDocument(filename.toOsSpecific()) + if not doc.LoadFile(): + return + + xconfig = doc.FirstChildElement('config') + if xconfig: + maxDiskUsage = xconfig.Attribute('max_disk_usage') + try: + self.maxDiskUsage = int(maxDiskUsage or '') + except ValueError: + pass + + + def checkDiskUsage(self): + """ Checks the total disk space used by all packages, and + removes old packages if necessary. """ + + totalSize = 0 + hosts = self.scanInstalledPackages() + for hostData in hosts: + for packageData in hostData.packages: + totalSize += packageData.totalSize + self.notify.info("Total Panda3D disk space used: %s MB" % ( + (totalSize + 524288) / 1048576)) + self.notify.info("Configured max usage is: %s MB" % ( + (self.maxDiskUsage + 524288) / 1048576)) + if totalSize <= self.maxDiskUsage: + # Still within budget; no need to clean up anything. + return + + # OK, we're over budget. Now we have to remove old packages. + usedPackages = [] + for hostData in hosts: + for packageData in hostData.packages: + if packageData.package and packageData.package.installed: + # Don't uninstall any packages we're currently using. + continue + + usedPackages.append((packageData.lastUse, packageData)) + + # Sort the packages into oldest-first order. + usedPackages.sort() + + # Delete packages until we free up enough space. + packages = [] + for lastUse, packageData in usedPackages: + if totalSize <= self.maxDiskUsage: + break + totalSize -= packageData.totalSize + + if packageData.package: + packages.append(packageData.package) + else: + # If it's an unknown package, just delete it directly. + print "Deleting unknown package %s" % (packageData.pathname) + shutil.rmtree(packageData.pathname.toOsSpecific(), True) + + packages = self.deletePackages(packages) + if packages: + print "Unable to delete %s packages" % (len(packages)) + + return + def stop(self): """ This method can be called by JavaScript to stop the application. """ @@ -473,6 +647,8 @@ class AppRunner(DirectObject): os.path.getsize = file.getsize sys.modules['glob'] = glob + self.checkDiskUsage() + def __startIfReady(self): """ Called internally to start the application. """ if self.started: @@ -557,6 +733,10 @@ class AppRunner(DirectObject): # The "super mirror" URL, generally used only by panda3d.exe. self.superMirrorUrl = superMirrorUrl + # Now that we have rootDir, we can read the config file. + self.readConfigXml() + + def addPackageInfo(self, name, platform, version, hostUrl, hostDir = None): """ Called by the browser for each one of the "required" packages that were preloaded before starting the application. diff --git a/direct/src/p3d/FileSpec.py b/direct/src/p3d/FileSpec.py index 4c12e77fb2..3696448f51 100644 --- a/direct/src/p3d/FileSpec.py +++ b/direct/src/p3d/FileSpec.py @@ -122,7 +122,7 @@ class FileSpec: return False if notify: - notify.debug("hash check ok: %s" % (pathname)) + notify.info("hash check ok: %s" % (pathname)) # The hash is OK after all. Change the file's timestamp back # to what we expect it to be, so we can quick-verify it @@ -164,7 +164,7 @@ class FileSpec: return False if notify: - notify.debug("hash check ok: %s" % (pathname)) + notify.info("hash check ok: %s" % (pathname)) # The hash is OK. If the timestamp is wrong, change it back # to what we expect it to be, so we can quick-verify it diff --git a/direct/src/p3d/HostInfo.py b/direct/src/p3d/HostInfo.py index a4fa0358f8..a4dbf1e735 100644 --- a/direct/src/p3d/HostInfo.py +++ b/direct/src/p3d/HostInfo.py @@ -4,6 +4,7 @@ from direct.p3d.PackageInfo import PackageInfo from direct.p3d.FileSpec import FileSpec from direct.directnotify.DirectNotifyGlobal import directNotify import time +import shutil class HostInfo: """ This class represents a particular download host serving up @@ -35,8 +36,8 @@ class HostInfo: the default is perPlatform = True. """ assert appRunner or rootDir or hostDir - - self.hostUrl = hostUrl + + self.__setHostUrl(hostUrl) self.appRunner = appRunner self.rootDir = rootDir if rootDir is None and appRunner: @@ -48,18 +49,6 @@ class HostInfo: if perPlatform is None: self.perPlatform = asMirror - # hostUrlPrefix is the host URL, but it is guaranteed to end - # with a slash. - self.hostUrlPrefix = hostUrl - if self.hostUrlPrefix[-1] != '/': - self.hostUrlPrefix += '/' - - # downloadUrlPrefix is the URL prefix that should be used for - # everything other than the contents.xml file. It might be - # the same as hostUrlPrefix, but in the case of an - # https-protected hostUrl, it will be the cleartext channel. - self.downloadUrlPrefix = self.hostUrlPrefix - # Initially false, this is set true when the contents file is # successfully read. self.hasContentsFile = False @@ -82,6 +71,27 @@ class HostInfo: # will be filled in when the contents file is read. self.packages = {} + def __setHostUrl(self, hostUrl): + """ Assigns self.hostUrl, and related values. """ + self.hostUrl = hostUrl + + if self.hostUrl is None: + # A special case: the URL will be set later. + self.hostUrlPrefix = None + self.downloadUrlPrefix = None + else: + # hostUrlPrefix is the host URL, but it is guaranteed to end + # with a slash. + self.hostUrlPrefix = hostUrl + if self.hostUrlPrefix[-1] != '/': + self.hostUrlPrefix += '/' + + # downloadUrlPrefix is the URL prefix that should be used for + # everything other than the contents.xml file. It might be + # the same as hostUrlPrefix, but in the case of an + # https-protected hostUrl, it will be the cleartext channel. + self.downloadUrlPrefix = self.hostUrlPrefix + def downloadContentsFile(self, http, redownload = False): """ Downloads the contents.xml file for this particular host, synchronously, and then reads it. Returns true on success, @@ -213,10 +223,14 @@ class HostInfo: return False # Look for our own entry in the hosts table. - self.__findHostXml(xcontents) + if self.hostUrl: + self.__findHostXml(xcontents) + else: + assert self.hostDir + self.__findHostXmlForHostDir(xcontents) if not self.hostDir: - self.__determineHostDir(None) + self.hostDir = self.__determineHostDir(None, self.hostUrl) # Get the list of packages available for download and/or import. xpackage = xcontents.FirstChildElement('package') @@ -275,6 +289,36 @@ class HostInfo: xhost = xhost.NextSiblingElement('host') + def __findHostXmlForHostDir(self, xcontents): + """ Looks for the or entry in the + contents.xml that corresponds to the host dir that we read the + contents.xml from. This is used when reading a contents.xml + file found on disk, as opposed to downloading it from a + site. """ + + xhost = xcontents.FirstChildElement('host') + while xhost: + url = xhost.Attribute('url') + hostDirBasename = xhost.Attribute('host_dir') + hostDir = self.__determineHostDir(hostDirBasename, url) + if hostDir == self.hostDir: + self.__setHostUrl(url) + self.readHostXml(xhost) + return + + xalthost = xhost.FirstChildElement('alt_host') + while xalthost: + url = xalthost.Attribute('url') + hostDirBasename = xalthost.Attribute('host_dir') + hostDir = self.__determineHostDir(hostDirBasename, url) + if hostDir == self.hostDir: + self.__setHostUrl(url) + self.readHostXml(xalthost) + return + xalthost = xalthost.NextSiblingElement('alt_host') + + xhost = xhost.NextSiblingElement('host') + def readHostXml(self, xhost): """ Reads a or entry and applies the data to this object. """ @@ -284,7 +328,8 @@ class HostInfo: self.descriptiveName = descriptiveName hostDirBasename = xhost.Attribute('host_dir') - self.__determineHostDir(hostDirBasename) + if not self.hostDir: + self.hostDir = self.__determineHostDir(hostDirBasename, self.hostUrl) # Get the "download" URL, which is the source from which we # download everything other than the contents.xml file. @@ -381,29 +426,85 @@ class HostInfo: return packages - def __determineHostDir(self, hostDirBasename): + def getAllPackages(self): + """ Returns a list of all available packages provided by this + host. """ + + result = [] + + items = self.packages.items() + items.sort() + for key, platforms in items: + if self.perPlatform: + # If we maintain a different answer per platform, + # return all of them. + pitems = platforms.items() + pitems.sort() + for pkey, package in pitems: + result.append(package) + else: + # If we maintain a a host for the current platform + # only (e.g. a client copy), then return only the + # current platform, or no particular platform. + package = platforms.get(PandaSystem.getPlatform(), None) + if not package: + package = platforms.get(None, None) + + if package: + result.append(package) + + return result + + def deletePackages(self, packages): + """ Removes all of the indicated packages from the disk, + uninstalling them and deleting all of their files. The + packages parameter must be a list of one or more PackageInfo + objects, for instance as returned by getPackage(). Returns + the list of packages that were NOT found. """ + + packages = packages[:] + + for key, platforms in self.packages.items(): + for platform, package in platforms.items(): + if package in packages: + self.__deletePackageFiles(package) + del platforms[platform] + packages.remove(package) + + if not platforms: + # If we've removed all the platforms for a given + # package, remove the key from the toplevel map. + del self.packages[key] + + return packages + + def __deletePackageFiles(self, package): + """ Called by deletePackage(), this actually removes the files + for the indicated package. """ + + self.notify.info("Deleting package %s: %s" % (package.packageName, package.getPackageDir())) + shutil.rmtree(package.getPackageDir().toOsSpecific(), True) + + def __determineHostDir(self, hostDirBasename, hostUrl): """ Hashes the host URL into a (mostly) unique directory string, which will be the root of the host's install tree. - Stores the resulting path, as a Filename, in self.hostDir. + Returns the resulting path, as a Filename. This code is duplicated in C++, in P3DHost::determine_host_dir(). """ - if self.hostDir: - return - if hostDirBasename: # If the contents.xml specified a host_dir parameter, use # it. - self.hostDir = self.rootDir + '/hosts' + hostDir = self.rootDir.cStr() + '/hosts' for component in hostDirBasename.split('/'): if component: if component[0] == '.': # Forbid ".foo" or "..". component = 'x' + component - self.hostDir += '/' - self.hostDir += component - return + hostDir += '/' + hostDir += component + return Filename(hostDir) hostDir = 'hosts/' @@ -414,22 +515,22 @@ class HostInfo: # We could use URLSpec, but we do it by hand instead, to make # it more likely that our hash code will exactly match the # similar logic in P3DHost. - p = self.hostUrl.find('://') + p = hostUrl.find('://') if p != -1: start = p + 3 - end = self.hostUrl.find('/', start) + end = hostUrl.find('/', start) # Now start .. end is something like "username@host:port". - at = self.hostUrl.find('@', start) + at = hostUrl.find('@', start) if at != -1 and at < end: start = at + 1 - colon = self.hostUrl.find(':', start) + colon = hostUrl.find(':', start) if colon != -1 and colon < end: end = colon # Now start .. end is just the hostname. - hostname = self.hostUrl[start : end] + hostname = hostUrl[start : end] # Now build a hash string of the whole URL. We'll use MD5 to # get a pretty good hash, with a minimum chance of collision. @@ -450,7 +551,8 @@ class HostInfo: keepHash = keepHash / 2; md = HashVal() - md.hashString(self.hostUrl) + md.hashString(hostUrl) hostDir += md.asHex()[:keepHash * 2] - self.hostDir = Filename(self.rootDir, hostDir) + hostDir = Filename(self.rootDir, hostDir) + return hostDir diff --git a/direct/src/p3d/InstalledHostData.py b/direct/src/p3d/InstalledHostData.py new file mode 100644 index 0000000000..470c300da3 --- /dev/null +++ b/direct/src/p3d/InstalledHostData.py @@ -0,0 +1,22 @@ +from pandac.PandaModules import URLSpec + +class InstalledHostData: + """ A list of instances of this class is returned by + AppRunner.scanInstalledPackages(). Each of these corresponds to a + particular host that has provided packages that have been + installed on the local client. """ + + def __init__(self, host, dirnode): + self.host = host + self.pathname = dirnode.pathname + self.totalSize = dirnode.getTotalSize() + self.packages = [] + + if self.host: + self.hostUrl = self.host.hostUrl + self.descriptiveName = self.host.descriptiveName + if not self.descriptiveName: + self.descriptiveName = URLSpec(self.hostUrl).getServer() + else: + self.hostUrl = 'unknown' + self.descriptiveName = 'unknown' diff --git a/direct/src/p3d/InstalledPackageData.py b/direct/src/p3d/InstalledPackageData.py new file mode 100644 index 0000000000..6d0675ac36 --- /dev/null +++ b/direct/src/p3d/InstalledPackageData.py @@ -0,0 +1,28 @@ +class InstalledPackageData: + """ A list of instances of this class is maintained by + InstalledHostData (which is in turn returned by + AppRunner.scanInstalledPackages()). Each of these corresponds to + a particular package that has been installed on the local + client. """ + + def __init__(self, package, dirnode): + self.package = package + self.pathname = dirnode.pathname + self.totalSize = dirnode.getTotalSize() + self.lastUse = None + + if self.package: + self.displayName = self.package.getFormattedName() + xusage = self.package.getUsage() + + if xusage: + lastUse = xusage.Attribute('last_use') + try: + lastUse = int(lastUse or '') + except ValueError: + lastUse = None + self.lastUse = lastUse + + else: + self.displayName = dirnode.pathname.getBasename() + diff --git a/direct/src/p3d/PackageInfo.py b/direct/src/p3d/PackageInfo.py index aebba9bd34..ea059395c0 100644 --- a/direct/src/p3d/PackageInfo.py +++ b/direct/src/p3d/PackageInfo.py @@ -1,6 +1,7 @@ -from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment, PStatCollector +from pandac.PandaModules import Filename, URLSpec, DocumentSpec, Ramfile, Multifile, Decompressor, EUOk, EUSuccess, VirtualFileSystem, Thread, getModelPath, ExecutionEnvironment, PStatCollector, TiXmlDocument, TiXmlDeclaration, TiXmlElement from pandac import PandaModules from direct.p3d.FileSpec import FileSpec +from direct.p3d.ScanDirectoryNode import ScanDirectoryNode from direct.showbase import VFSImporter from direct.directnotify.DirectNotifyGlobal import directNotify from direct.task.TaskManagerGlobal import taskMgr @@ -8,6 +9,7 @@ import os import sys import random import time +import copy class PackageInfo: @@ -32,6 +34,8 @@ class PackageInfo: restartDownload = 3 stepContinue = 4 + UsageBasename = 'usage.xml' + class InstallStep: """ This class is one step of the installPlan list; it represents a single atomic piece of the installation step, and @@ -117,6 +121,11 @@ class PackageInfo: # meaning it's been added to the paths and all. self.installed = False + # This is set true when the package has been updated in this + # session, but not yet written to usage.xml. + self.updated = False + self.diskSpace = None + def getPackageDir(self): """ Returns the directory in which this package is installed. This may not be known until the host's contents.xml file has @@ -493,6 +502,7 @@ class PackageInfo: contents = self.__scanDirectoryRecursively(self.getPackageDir()) self.__removeFileFromList(contents, self.descFileBasename) self.__removeFileFromList(contents, self.compressedArchive.filename) + self.__removeFileFromList(contents, self.UsageBasename) if not self.asMirror: self.__removeFileFromList(contents, self.uncompressedArchive.filename) for file in self.extracts: @@ -507,6 +517,7 @@ class PackageInfo: self.notify.info("Removing %s" % (filename)) pathname = Filename(self.getPackageDir(), filename) pathname.unlink() + self.updated = True if self.asMirror: return self.compressedArchive.quickVerify(self.getPackageDir(), notify = self.notify) @@ -681,6 +692,8 @@ class PackageInfo: packageDir. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ + self.updated = True + if not urlbase: urlbase = self.descFileDirname + '/' + fileSpec.filename @@ -817,6 +830,8 @@ class PackageInfo: operation. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ + self.updated = True + origPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) patchPathname = Filename(self.getPackageDir(), patchfile.file.filename) result = Filename.temporary('', 'patch_') @@ -857,6 +872,8 @@ class PackageInfo: archive. Yields one of stepComplete, stepFailed, restartDownload, or stepContinue. """ + self.updated = True + sourcePathname = Filename(self.getPackageDir(), self.compressedArchive.filename) targetPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) targetPathname.unlink() @@ -905,6 +922,8 @@ class PackageInfo: self.hasPackage = True yield self.stepComplete; return + self.updated = True + mfPathname = Filename(self.getPackageDir(), self.uncompressedArchive.filename) self.notify.info("Unpacking %s" % (mfPathname)) mf = Multifile() @@ -1026,4 +1045,94 @@ class PackageInfo: self.installed = True appRunner.installedPackages.append(self) + self.markUsed() + return True + + def __measureDiskSpace(self): + """ Returns the amount of space used by this package, in + bytes, as determined by examining the actual contents of the + package directory and its subdirectories. """ + + thisDir = ScanDirectoryNode(self.getPackageDir(), ignoreUsageXml = True) + diskSpace = thisDir.getTotalSize() + self.notify.info("Package %s uses %s MB" % ( + self.packageName, (diskSpace + 524288) / 1048576)) + return diskSpace + + def markUsed(self): + """ Marks the package as having been used. This is normally + called automatically by installPackage(). """ + + if not hasattr(PandaModules, 'TiXmlDocument'): + return + + if self.updated: + # If we've just installed a new version of the package, + # re-measure the actual disk space used. + self.diskSpace = self.__measureDiskSpace() + + filename = Filename(self.getPackageDir(), self.UsageBasename) + doc = TiXmlDocument(filename.toOsSpecific()) + if not doc.LoadFile(): + decl = TiXmlDeclaration("1.0", "utf-8", "") + doc.InsertEndChild(decl) + + xusage = doc.FirstChildElement('usage') + if not xusage: + doc.InsertEndChild(TiXmlElement('usage')) + xusage = doc.FirstChildElement('usage') + + now = int(time.time()) + + count = xusage.Attribute('count_app') + try: + count = int(count or '') + except ValueError: + count = 0 + xusage.SetAttribute('first_use', str(now)) + count += 1 + xusage.SetAttribute('count_app', str(count)) + + xusage.SetAttribute('last_use', str(now)) + + if self.updated: + xusage.SetAttribute('last_update', str(now)) + self.updated = False + else: + # Since we haven't changed the disk space, we can just + # read it from the previous xml file. + diskSpace = xusage.Attribute('disk_space') + try: + diskSpace = int(diskSpace or '') + except ValueError: + # Unless it wasn't set already. + self.diskSpace = self.__measureDiskSpace() + + xusage.SetAttribute('disk_space', str(self.diskSpace)) + + # Write the file to a temporary filename, then atomically move + # it to its actual filename, to avoid race conditions when + # updating this file. + tfile = Filename.temporary(self.getPackageDir().cStr(), '.xml') + if doc.SaveFile(tfile.toOsSpecific()): + tfile.renameTo(filename) + + def getUsage(self): + """ Returns the xusage element that is read from the usage.xml + file, or None if there is no usage.xml file. """ + + if not hasattr(PandaModules, 'TiXmlDocument'): + return None + + filename = Filename(self.getPackageDir(), self.UsageBasename) + doc = TiXmlDocument(filename.toOsSpecific()) + if not doc.LoadFile(): + return None + + xusage = doc.FirstChildElement('usage') + if not xusage: + return None + + return copy.copy(xusage) + diff --git a/direct/src/p3d/ScanDirectoryNode.py b/direct/src/p3d/ScanDirectoryNode.py new file mode 100644 index 0000000000..21df0d3eee --- /dev/null +++ b/direct/src/p3d/ScanDirectoryNode.py @@ -0,0 +1,90 @@ +from pandac.PandaModules import VirtualFileSystem, VirtualFileMountSystem, Filename, TiXmlDocument +vfs = VirtualFileSystem.getGlobalPtr() + +class ScanDirectoryNode: + """ This class is used to scan a list of files on disk. """ + + def __init__(self, pathname, ignoreUsageXml = False): + self.pathname = pathname + self.filenames = [] + self.fileSize = 0 + self.nested = [] + self.nestedSize = 0 + + xusage = None + if not ignoreUsageXml: + # Look for a usage.xml file in this directory. If we find + # one, we read it for the file size and then stop here, as + # an optimization. + usageFilename = Filename(pathname, 'usage.xml') + doc = TiXmlDocument(usageFilename.toOsSpecific()) + if doc.LoadFile(): + xusage = doc.FirstChildElement('usage') + if xusage: + diskSpace = xusage.Attribute('disk_space') + try: + diskSpace = int(diskSpace or '') + except ValueError: + diskSpace = None + if diskSpace is not None: + self.fileSize = diskSpace + return + + for vfile in vfs.scanDirectory(self.pathname): + if hasattr(vfile, 'getMount'): + if not isinstance(vfile.getMount(), VirtualFileMountSystem): + # Not a real file; ignore it. + continue + + if vfile.isDirectory(): + # A nested directory. + subdir = ScanDirectoryNode(vfile.getFilename(), ignoreUsageXml = ignoreUsageXml) + self.nested.append(subdir) + self.nestedSize += subdir.getTotalSize() + + elif vfile.isRegularFile(): + # A nested file. + self.filenames.append(vfile.getFilename()) + self.fileSize += vfile.getFileSize() + + else: + # Some other wacky file thing. + self.filenames.append(vfile.getFilename()) + + if xusage: + # Now update the usage.xml file with the newly-determined + # disk space. + xusage.SetAttribute('disk_space', str(self.getTotalSize())) + tfile = Filename.temporary(pathname.cStr(), '.xml') + if doc.SaveFile(tfile.toOsSpecific()): + tfile.renameTo(usageFilename) + + def getTotalSize(self): + return self.nestedSize + self.fileSize + + def extractSubdir(self, pathname): + """ Finds the ScanDirectoryNode within this node that + corresponds to the indicated full pathname. If it is found, + removes it from its parent, and returns it. If it is not + found, returns None. """ + + # We could be a little smarter here, but why bother. Just + # recursively search all children. + for subdir in self.nested: + if subdir.pathname == pathname: + self.nested.remove(subdir) + self.nestedSize -= subdir.getTotalSize() + return subdir + + result = subdir.extractSubdir(pathname) + if result: + self.nestedSize -= result.getTotalSize() + if subdir.getTotalSize() == 0: + # No other files in the subdirectory that contains + # this package; remove it too. + self.nested.remove(subdir) + return result + + return None + + diff --git a/direct/src/plugin/find_root_dir.cxx b/direct/src/plugin/find_root_dir.cxx index 58805eb3d1..3d661152a6 100755 --- a/direct/src/plugin/find_root_dir.cxx +++ b/direct/src/plugin/find_root_dir.cxx @@ -13,6 +13,8 @@ //////////////////////////////////////////////////////////////////// #include "find_root_dir.h" +#include "mkdir_complete.h" +#include "get_tinyxml.h" #ifdef _WIN32 #include @@ -35,27 +37,6 @@ DEFINE_KNOWN_FOLDER(FOLDERID_InternetCache, 0x352481E8, 0x33BE, 0x4251, 0xBA, 0x #endif // _WIN32 -#ifdef _WIN32 -//////////////////////////////////////////////////////////////////// -// Function: check_root_dir -// Description: Tests the proposed root dir string for validity. -// Returns true if Panda3D can be successfully created -// within the dir, false otherwise. -//////////////////////////////////////////////////////////////////// -static bool -check_root_dir(const string &root) { - // Attempt to make it first, if possible. - CreateDirectory(root.c_str(), NULL); - - bool isdir = false; - DWORD results = GetFileAttributes(root.c_str()); - if (results != -1) { - isdir = (results & FILE_ATTRIBUTE_DIRECTORY) != 0; - } - return isdir; -} -#endif // _WIN32 - #ifdef _WIN32 //////////////////////////////////////////////////////////////////// // Function: get_csidl_dir @@ -71,7 +52,7 @@ get_csidl_dir(int csidl) { string root = buffer; root += string("/Panda3D"); - if (check_root_dir(root)) { + if (mkdir_complete(root, cerr)) { return root; } } @@ -109,12 +90,12 @@ wstr_to_string(string &result, const LPWSTR wstr) { //////////////////////////////////////////////////////////////////// -// Function: find_root_dir -// Description: Returns the path to the installable Panda3D directory -// on the user's machine. +// Function: find_root_dir_default +// Description: Returns the path to the system-default for the root +// directory. This is where we look first. //////////////////////////////////////////////////////////////////// -string -find_root_dir() { +static string +find_root_dir_default() { #ifdef _WIN32 // First, use IEIsProtectedModeProcess() to determine if we are // running in IE's "protected mode" under Vista. @@ -165,7 +146,7 @@ find_root_dir() { CoTaskMemFree(cache_path); root += string("/Panda3D"); - if (check_root_dir(root)) { + if (mkdir_complete(root, cerr)) { FreeLibrary(shell32); FreeLibrary(ieframe); return root; @@ -199,7 +180,7 @@ find_root_dir() { } else { CoTaskMemFree(cache_path); root += string("/Panda3D"); - if (check_root_dir(root)) { + if (mkdir_complete(root, cerr)) { FreeLibrary(ieframe); return root; } @@ -236,7 +217,7 @@ find_root_dir() { if (GetTempPath(buffer_size, buffer) != 0) { root = buffer; root += string("Panda3D"); - if (check_root_dir(root)) { + if (mkdir_complete(root, cerr)) { return root; } } @@ -273,3 +254,42 @@ find_root_dir() { return "."; } + +//////////////////////////////////////////////////////////////////// +// Function: find_root_dir +// Description: Returns the path to the installable Panda3D directory +// on the user's machine. +//////////////////////////////////////////////////////////////////// +string +find_root_dir() { + string root = find_root_dir_default(); + + // Now look for a config.xml file in that directory, which might + // redirect us elsewhere. + string config_filename = root + "/config.xml"; + TiXmlDocument doc(config_filename); + if (!doc.LoadFile()) { + // No config.xml found, or not valid xml. + return root; + } + + TiXmlElement *xconfig = doc.FirstChildElement("config"); + if (xconfig == NULL) { + // No element within config.xml. + return root; + } + + const char *new_root = xconfig->Attribute("root_dir"); + if (new_root == NULL || *new_root == '\0') { + // No root_dir specified. + return root; + } + + if (!mkdir_complete(new_root, cerr)) { + // The specified root_dir wasn't valid. + return root; + } + + // We've been redirected to another location. Respect that. + return new_root; +} diff --git a/direct/src/plugin/p3dHost.cxx b/direct/src/plugin/p3dHost.cxx index c14be5b177..7eb588dca7 100644 --- a/direct/src/plugin/p3dHost.cxx +++ b/direct/src/plugin/p3dHost.cxx @@ -394,6 +394,26 @@ get_package_desc_file(FileSpec &desc_file, // out return false; } +//////////////////////////////////////////////////////////////////// +// Function: P3DHost::forget_package +// Access: Public +// Description: Removes the indicated package from the cache of +// packages known by this host. This is invoked from +// the Python side by AppRunner.deletePackages(), so +// that we remove the package before deleting its files. +//////////////////////////////////////////////////////////////////// +void P3DHost:: +forget_package(P3DPackage *package, const string &alt_host) { + string key = package->get_package_name() + "_" + package->get_package_version(); + + PackageMap &package_map = _packages[alt_host]; + + // Hmm, this is a memory leak. But we allow it to remain, since + // it's an unusual circumstance (uninstalling), and it's safer to + // leak than to risk a floating pointer. + package_map.erase(key); +} + //////////////////////////////////////////////////////////////////// // Function: P3DHost::migrate_package // Access: Public @@ -494,6 +514,7 @@ uninstall() { P3DInstanceManager *inst_mgr = P3DInstanceManager::get_global_ptr(); inst_mgr->delete_directory_recursively(_host_dir); + inst_mgr->forget_host(this); } //////////////////////////////////////////////////////////////////// diff --git a/direct/src/plugin/p3dHost.h b/direct/src/plugin/p3dHost.h index 29fc14439a..ebddf4a2e1 100644 --- a/direct/src/plugin/p3dHost.h +++ b/direct/src/plugin/p3dHost.h @@ -59,6 +59,7 @@ public: const string &package_name, const string &package_version); + void forget_package(P3DPackage *package, const string &alt_host = ""); void migrate_package(P3DPackage *package, const string &alt_host, P3DHost *new_host); void choose_random_mirrors(vector &result, int num_mirrors); diff --git a/direct/src/plugin/p3dInstance.cxx b/direct/src/plugin/p3dInstance.cxx index 5e58a7f16f..bee7dfa2a3 100644 --- a/direct/src/plugin/p3dInstance.cxx +++ b/direct/src/plugin/p3dInstance.cxx @@ -1473,8 +1473,8 @@ void P3DInstance:: uninstall_host() { uninstall_packages(); + // Collect the set of hosts referenced by this instance. set hosts; - Packages::const_iterator pi; for (pi = _packages.begin(); pi != _packages.end(); ++pi) { P3DPackage *package = (*pi); @@ -1484,6 +1484,7 @@ uninstall_host() { } } + // Uninstall all of them. set::iterator hi; for (hi = hosts.begin(); hi != hosts.end(); ++hi) { P3DHost *host = (*hi); @@ -3098,6 +3099,7 @@ report_package_done(P3DPackage *package, bool success) { nout << "No entry in image package\n"; return; } + package->mark_used(); for (int i = 0; i < (int)IT_none; ++i) { if (_image_files[i]._use_standard_image) { @@ -3129,6 +3131,8 @@ report_package_done(P3DPackage *package, bool success) { // failing to download it) means we can finish checking the // authenticity of the p3d file. + package->mark_used(); + // Take down the download progress. if (_splash_window != NULL) { _splash_window->set_install_progress(0.0, true, 0); @@ -3145,6 +3149,8 @@ report_package_done(P3DPackage *package, bool success) { // Another special case: successfully downloading p3dcert means we // can enable the auth button. + package->mark_used(); + // Take down the download progress. if (_splash_window != NULL) { _splash_window->set_install_progress(0.0, true, 0); diff --git a/direct/src/plugin/p3dInstanceManager.cxx b/direct/src/plugin/p3dInstanceManager.cxx index 8f5ab57def..fd21c7d301 100644 --- a/direct/src/plugin/p3dInstanceManager.cxx +++ b/direct/src/plugin/p3dInstanceManager.cxx @@ -614,6 +614,21 @@ get_host(const string &host_url) { return host; } +//////////////////////////////////////////////////////////////////// +// Function: P3DInstanceManager::forget_host +// Access: Public +// Description: Removes the indicated host from the cache. +//////////////////////////////////////////////////////////////////// +void P3DInstanceManager:: +forget_host(P3DHost *host) { + const string &host_url = host->get_host_url(); + + // Hmm, this is a memory leak. But we allow it to remain, since + // it's an unusual circumstance (uninstalling), and it's safer to + // leak than to risk a floating pointer. + _hosts.erase(host_url); +} + //////////////////////////////////////////////////////////////////// // Function: P3DInstanceManager::get_unique_id // Access: Public @@ -1395,8 +1410,9 @@ nt_thread_run() { P3DInstance *inst = (*ni); assert(inst != NULL); P3D_request_ready_func *func = inst->get_request_ready_func(); - assert(func != NULL); - (*func)(inst); + if (func != NULL) { + (*func)(inst); + } } _notify_ready.acquire(); } diff --git a/direct/src/plugin/p3dInstanceManager.h b/direct/src/plugin/p3dInstanceManager.h index 40618a9bd0..0fe74958dc 100644 --- a/direct/src/plugin/p3dInstanceManager.h +++ b/direct/src/plugin/p3dInstanceManager.h @@ -108,6 +108,7 @@ public: void wait_request(double timeout); P3DHost *get_host(const string &host_url); + void forget_host(P3DHost *host); inline int get_num_instances() const; diff --git a/direct/src/plugin/p3dPackage.cxx b/direct/src/plugin/p3dPackage.cxx index 59cecc18a9..ab26acf596 100755 --- a/direct/src/plugin/p3dPackage.cxx +++ b/direct/src/plugin/p3dPackage.cxx @@ -72,6 +72,7 @@ P3DPackage(P3DHost *host, const string &package_name, _failed = false; _active_download = NULL; _saved_download = NULL; + _updated = false; } //////////////////////////////////////////////////////////////////// @@ -210,6 +211,70 @@ remove_instance(P3DInstance *inst) { begin_info_download(); } +//////////////////////////////////////////////////////////////////// +// Function: P3DPackage::mark_used +// Access: Public +// Description: Marks this package as having been "used", for +// accounting purposes. +//////////////////////////////////////////////////////////////////// +void P3DPackage:: +mark_used() { + // Unlike the Python variant of this function, we don't mess around + // with updating the disk space or anything. + string filename = get_package_dir() + "/usage.xml"; + TiXmlDocument doc(filename); + if (!doc.LoadFile()) { + TiXmlDeclaration *decl = new TiXmlDeclaration("1.0", "utf-8", ""); + doc.LinkEndChild(decl); + } + + TiXmlElement *xusage = doc.FirstChildElement("usage"); + if (xusage == NULL) { + xusage = new TiXmlElement("usage"); + doc.LinkEndChild(xusage); + } + + time_t now = time(NULL); + int count = 0; + xusage->Attribute("count_runtime", &count); + if (count == 0) { + xusage->SetAttribute("first_use", now); + } + + ++count; + xusage->SetAttribute("count_runtime", count); + xusage->SetAttribute("last_use", now); + + if (_updated) { + // If we've updated the package, we're no longer sure what its + // disk space is. Remove that from the XML file, so that the + // Python code can recompute it later. + xusage->RemoveAttribute("disk_space"); + xusage->SetAttribute("last_update", now); + } + + // Write the file to a temporary filename, then atomically move it + // to its actual filename, to avoid race conditions. + ostringstream strm; + strm << get_package_dir() << "/usage_"; +#ifdef _WIN32 + strm << GetCurrentProcessId(); +#else + strm << getpid(); +#endif + strm << ".xml"; + string tfile = strm.str(); + + unlink(tfile.c_str()); + if (doc.SaveFile(tfile)) { + if (rename(tfile.c_str(), filename.c_str()) != 0) { + // If rename failed, remove the original file first. + unlink(filename.c_str()); + rename(tfile.c_str(), filename.c_str()); + } + } +} + //////////////////////////////////////////////////////////////////// // Function: P3DPackage::uninstall // Access: Public @@ -247,6 +312,9 @@ uninstall() { _ready = false; _failed = false; _allow_data_download = false; + + // Make sure the host forgets us too. + _host->forget_package(this); } //////////////////////////////////////////////////////////////////// @@ -707,6 +775,7 @@ got_desc_file(TiXmlDocument *doc, bool freshly_downloaded) { inst_mgr->remove_file_from_list(contents, _desc_file_basename); inst_mgr->remove_file_from_list(contents, _uncompressed_archive.get_filename()); + inst_mgr->remove_file_from_list(contents, "usage.xml"); Extracts::iterator ei; for (ei = _extracts.begin(); ei != _extracts.end(); ++ei) { inst_mgr->remove_file_from_list(contents, (*ei).get_filename()); @@ -727,6 +796,7 @@ got_desc_file(TiXmlDocument *doc, bool freshly_downloaded) { chmod(pathname.c_str(), 0644); #endif unlink(pathname.c_str()); + _updated = true; } // Verify the uncompressed archive. @@ -1509,6 +1579,7 @@ do_step(bool download_finished) { if (_download->get_download_success()) { // The Download object has already validated the hash. + _package->_updated = true; return IT_step_complete; } else if (_download->get_download_terminated()) { @@ -1801,6 +1872,7 @@ thread_step() { unlink(source_pathname.c_str()); // All done uncompressing. + _package->_updated = true; return IT_step_complete; } @@ -1847,6 +1919,7 @@ thread_step() { return IT_step_failed; } + _package->_updated = true; return IT_step_complete; } @@ -1901,6 +1974,7 @@ thread_step() { return IT_step_failed; } + _package->_updated = true; return IT_step_complete; } diff --git a/direct/src/plugin/p3dPackage.h b/direct/src/plugin/p3dPackage.h index 93e58708cb..4caa89dbe1 100755 --- a/direct/src/plugin/p3dPackage.h +++ b/direct/src/plugin/p3dPackage.h @@ -69,6 +69,7 @@ public: void add_instance(P3DInstance *inst); void remove_instance(P3DInstance *inst); + void mark_used(); void uninstall(); TiXmlElement *make_xml(); @@ -290,6 +291,7 @@ private: size_t _unpack_size; Extracts _extracts; + bool _updated; static const double _download_factor; static const double _uncompress_factor; diff --git a/direct/src/plugin/p3dSession.cxx b/direct/src/plugin/p3dSession.cxx index 882d5b0b28..ff8a5cec37 100644 --- a/direct/src/plugin/p3dSession.cxx +++ b/direct/src/plugin/p3dSession.cxx @@ -958,6 +958,8 @@ start_p3dpython(P3DInstance *inst) { _env += string("_ROOT="); _env += package->get_package_dir(); _env += '\0'; + + package->mark_used(); } // Check for a few tokens that have special meaning at this level.