use lipo to support universal binaries (mostly) automatically

direct/src/p3d/Packager.py

@@ -141,16 +141,17 @@ class Packager:
             return False
                 
     class ExcludeFilename:
-        def __init__(self, filename, caseSensitive):
+        def __init__(self, packager, filename, caseSensitive):
+            self.packager = packager
             self.localOnly = (not filename.getDirname())
             if not self.localOnly:
                 filename = Filename(filename)
                 filename.makeCanonical()
             self.glob = GlobPattern(filename.cStr())
 
-            if PandaSystem.getPlatform().startswith('win'):
+            if self.packager.platform.startswith('win'):
                 self.glob.setCaseSensitive(False)
-            elif PandaSystem.getPlatform().startswith('osx'):
+            elif self.packager.platform.startswith('osx'):
                 self.glob.setCaseSensitive(False)
 
         def matches(self, filename):
@@ -293,8 +294,15 @@ class Packager:
             self.packageName = packageName
             self.packager = packager
             self.notify = packager.notify
-            
+
+            # The platform is initially None until we know the file is
+            # platform-specific.
             self.platform = None
+
+            # The arch string, though, is pre-loaded from the system
+            # arch string, so we can sensibly call otool.
+            self.arch = self.packager.arch
+
             self.version = None
             self.host = None
             self.p3dApplication = False
@@ -326,7 +334,7 @@ class Packager:
 
             # This records the current list of modules we have added so
             # far.
-            self.freezer = FreezeTool.Freezer()
+            self.freezer = FreezeTool.Freezer(platform = self.packager.platform)
 
         def close(self):
             """ Writes out the contents of the current package. """
@@ -414,7 +422,11 @@ class Packager:
 
             if platformSpecific and self.platformSpecificConfig is not False:
                 if not self.platform:
-                    self.platform = PandaSystem.getPlatform()
+                    self.platform = self.packager.platform
+
+            if self.platform and self.platform.startswith('osx_'):
+                # Get the OSX "arch" specification.
+                self.arch = self.platform[4:]
             
 
         def installMultifile(self):
@@ -518,9 +530,9 @@ class Packager:
                 self.components.append(('m', newName.lower(), xmodule))
 
             # Now look for implicit shared-library dependencies.
-            if PandaSystem.getPlatform().startswith('win'):
+            if self.packager.platform.startswith('win'):
                 self.__addImplicitDependenciesWindows()
-            elif PandaSystem.getPlatform().startswith('osx'):
+            elif self.packager.platform.startswith('osx'):
                 self.__addImplicitDependenciesOSX()
             else:
                 self.__addImplicitDependenciesPosix()
@@ -752,7 +764,7 @@ class Packager:
         def excludeFile(self, filename):
             """ Excludes the named file (or glob pattern) from the
             package. """
-            xfile = Packager.ExcludeFilename(filename, self.packager.caseSensitive)
+            xfile = Packager.ExcludeFilename(self.packager, filename, self.packager.caseSensitive)
             self.excludedFilenames.append(xfile)
 
         def __addImplicitDependenciesWindows(self):
@@ -910,13 +922,17 @@ class Packager:
                     continue
 
                 tempFile = Filename.temporary('', 'p3d_', '.txt')
-                command = '/usr/bin/otool -L "%s" >"%s"' % (
+                command = '/usr/bin/otool -arch all -L "%s" >"%s"' % (
                     file.filename.toOsSpecific(),
                     tempFile.toOsSpecific())
-                try:
-                    os.system(command)
-                except:
-                    pass
+                if self.arch:
+                    command = '/usr/bin/otool -arch %s -L "%s" >"%s"' % (
+                        self.arch,
+                        file.filename.toOsSpecific(),
+                        tempFile.toOsSpecific())
+                exitStatus = os.system(command)
+                if exitStatus != 0:
+                    self.notify.warning('Command failed: %s' % (command))
                 filenames = None
 
                 if tempFile.exists():
@@ -1619,7 +1635,13 @@ class Packager:
                 stream = StringStream(file.text)
                 self.multifile.addSubfile(file.newName, stream, compressionLevel)
                 self.multifile.flush()
+
+            elif file.executable and self.arch:
+                if not self.__addOsxExecutable(file):
+                    return
+
             else:
+                # Copy an ordinary file into the multifile.
                 self.multifile.addSubfile(file.newName, file.filename, compressionLevel)
             if file.extract:
                 if file.text:
@@ -1639,6 +1661,72 @@ class Packager:
             xcomponent.SetAttribute('filename', file.newName)
             self.components.append(('c', file.newName.lower(), xcomponent))
 
+        def __addOsxExecutable(self, file):
+            """ Adds an executable or shared library to the multifile,
+            with respect to OSX's fat-binary features.  Returns true
+            on success, false on failure. """
+
+            compressionLevel = 0
+            if file.compress:
+                compressionLevel = self.compressionLevel
+
+            # If we're on OSX and adding only files for a
+            # particular architecture, use lipo to strip out the
+            # part of the file for that architecture.
+
+            # First, we need to verify that it is in fact a
+            # universal binary.
+            tfile = Filename.temporary('', 'p3d_')
+            command = '/usr/bin/lipo -info "%s" >"%s"' % (
+                file.filename.toOsSpecific(),
+                tfile.toOsSpecific())
+            exitStatus = os.system(command)
+            if exitStatus != 0:
+                self.notify.warning("Not an executable file: %s" % (file.filename))
+                # Just add it anyway.
+                self.multifile.addSubfile(file.newName, file.filename, compressionLevel)
+                return True
+
+            # The lipo command succeeded, so it really is an
+            # executable file.  Parse the lipo output to figure out
+            # which architectures the file supports.
+            arches = []
+            lipoData = open(tfile.toOsSpecific(), 'r').read()
+            tfile.unlink()
+            if ':' in lipoData:
+                arches = lipoData.rsplit(':', 1)[1]
+                arches = arches.split()
+
+            if arches == [self.arch]:
+                # The file only contains the one architecture that
+                # we want anyway.
+                self.multifile.addSubfile(file.newName, file.filename, compressionLevel)
+                return True
+            
+            if self.arch not in arches:
+                # The file doesn't support the architecture that we
+                # want at all.  Omit the file.
+                self.notify.warning("%s doesn't support architecture %s" % (
+                    file.filename, self.arch))
+                return False
+
+
+            # The file contains multiple architectures.  Get
+            # out just the one we want.
+            command = '/usr/bin/lipo -thin %s -output "%s" "%s"' % (
+                self.arch, tfile.toOsSpecific(),
+                file.filename.toOsSpecific())
+            exitStatus = os.system(command)
+            if exitStatus != 0:
+                self.notify.error('Command failed: %s' % (command))
+            self.multifile.addSubfile(file.newName, tfile, compressionLevel)
+            if file.deleteTemp:
+                file.filename.unlink()
+            file.filename = tfile
+            file.deleteTemp = True
+            return True
+            
+
         def requirePackage(self, package):
             """ Indicates a dependency on the given package.  This
             also implicitly requires all of the package's requirements
@@ -1655,11 +1743,14 @@ class Packager:
                     self.skipModules[moduleName] = mdef
 
     # Packager constructor
-    def __init__(self):
+    def __init__(self, platform = None):
 
         # The following are config settings that the caller may adjust
         # before calling any of the command methods.
 
+        # The platform string.
+        self.setPlatform(platform)
+
         # This should be set to a Filename.
         self.installDir = None
 
@@ -1670,13 +1761,13 @@ class Packager:
         self.addHost(self.host)
 
         # A search list for previously-built local packages.
-        self.installSearch = ConfigVariableSearchPath('pdef-path')
+        self.installSearch = list(ConfigVariableSearchPath('pdef-path').getDirectories())
 
         # The system PATH, for searching dll's and exe's.
         self.executablePath = DSearchPath()
-        if PandaSystem.getPlatform().startswith('win'):
+        if self.platform.startswith('win'):
             self.addWindowsSearchPath(self.executablePath, "PATH")
-        elif PandaSystem.getPlatform().startswith('osx'):
+        elif self.platform.startswith('osx'):
             self.addPosixSearchPath(self.executablePath, "DYLD_LIBRARY_PATH")
             self.addPosixSearchPath(self.executablePath, "LD_LIBRARY_PATH")
             self.addPosixSearchPath(self.executablePath, "PATH")
@@ -1690,6 +1781,7 @@ class Packager:
             self.executablePath.appendDirectory('/usr/lib')
             self.executablePath.appendDirectory('/usr/local/lib')
         
+        import platform
         if platform.uname()[1]=="pcbsd":
             self.executablePath.appendDirectory('/usr/PCBSD/local/lib')
 
@@ -1702,9 +1794,6 @@ class Packager:
         # generated.
         self.signParams = []
 
-        # The platform string.
-        self.platform = PandaSystem.getPlatform()
-
         # Optional signing and encrypting features.
         self.encryptionKey = None
         self.prcEncryptionKey = None
@@ -1729,9 +1818,9 @@ class Packager:
 
         # Is this file system case-sensitive?
         self.caseSensitive = True
-        if PandaSystem.getPlatform().startswith('win'):
+        if self.platform.startswith('win'):
             self.caseSensitive = False
-        elif PandaSystem.getPlatform().startswith('osx'):
+        elif self.platform.startswith('osx'):
             self.caseSensitive = False
 
         # Get the list of filename extensions that are recognized as
@@ -1847,6 +1936,24 @@ class Packager:
         # file.
         self.contents = {}
 
+    def setPlatform(self, platform = None):
+        """ Sets the platform that this Packager will compute for.  On
+        OSX, this can be used to specify the particular architecture
+        we are building; on other platforms, it is probably a mistake
+        to set this.
+
+        You should call this before doing anything else with the
+        Packager.  It's even better to pass the platform string to the
+        constructor.  """
+
+        self.platform = platform or PandaSystem.getPlatform()
+
+        # OSX uses this "arch" string for the otool and lipo commands.
+        self.arch = None
+        if self.platform.startswith('osx_'):
+            self.arch = self.platform[4:]
+        
+
     def setHost(self, host, downloadUrl = None,
                 descriptiveName = None, hostDir = None,
                 mirrors = None):
@@ -1989,7 +2096,7 @@ class Packager:
         globals = {}
         globals['__name__'] = packageDef.getBasenameWoExtension()
 
-        globals['platform'] = PandaSystem.getPlatform()
+        globals['platform'] = self.platform
         globals['packager'] = self
 
         # We'll stuff all of the predefined functions, and the
@@ -2176,7 +2283,7 @@ class Packager:
             return package
 
         # Look on the searchlist.
-        for dirname in self.installSearch.getDirectories():
+        for dirname in self.installSearch:
             package = self.__scanPackageDir(dirname, packageName, platform or self.platform, version, host, requires = requires)
             if not package:
                 package = self.__scanPackageDir(dirname, packageName, platform, version, host, requires = requires)
@@ -2590,7 +2697,7 @@ class Packager:
             self.do_file('p3dcert.exe')
 
         self.do_file('p3dpython.exe')
-        if PandaSystem.getPlatform().startswith('win'):
+        if self.platform.startswith('win'):
             self.do_file('p3dpythonw.exe')
         self.do_file('libp3dpython.dll')
 
@@ -2640,7 +2747,7 @@ class Packager:
                         deleteTemp = True, explicit = True, extract = True)
         package.addExtensionModules()
         if not package.platform:
-            package.platform = PandaSystem.getPlatform()
+            package.platform = self.platform
 
         # Reset the freezer for more Python files.
         freezer.reset()

direct/src/p3d/ppackage.py

@@ -79,10 +79,23 @@ Options:
      initially, but should not be set on an application intended for
      deployment.
 
+  -u
+     On the Mac OSX platform, this means that Panda was built with
+     universal binaries, and the package should be built that way as
+     well (that is, a version of the the package should be created for
+     each supported architecture).  On other platforms, this option
+     does nothing.  This is therefore safe to apply in all cases, if
+     you wish to take advantage of universal binaries.  This is
+     equivalent to "-P osx_ppc -P osx_i386" on Mac platforms.
+
   -P platform
      Specify the platform to masquerade as.  The default is whatever
-     platform Panda has been built for.  It is probably unwise to set
-     this, unless you know what you are doing.
+     platform Panda has been built for.  You can use this on Mac OSX
+     in order to build packages for an alternate architecture if you
+     have built Panda with universal binaries; you may repeat this
+     option for each architecture you wish to support.  For other
+     platforms, it is probably a mistake to set this.  However, see
+     the option -u.
 
   -h
      Display this help
@@ -100,31 +113,38 @@ def usage(code, msg = ''):
     print >> sys.stderr, msg
     sys.exit(code)
 
-packager = Packager.Packager()
+installDir = None
 buildPatches = False
+installSearch = []
+signParams = []
+allowPythonDev = False
+universalBinaries = False
+platforms = []
 
 try:
-    opts, args = getopt.getopt(sys.argv[1:], 'i:ps:S:DP:h')
+    opts, args = getopt.getopt(sys.argv[1:], 'i:ps:S:DuP:h')
 except getopt.error, msg:
     usage(1, msg)
 
 for opt, arg in opts:
     if opt == '-i':
-        packager.installDir = Filename.fromOsSpecific(arg)
+        installDir = Filename.fromOsSpecific(arg)
     elif opt == '-p':
         buildPatches = True
     elif opt == '-s':
-        packager.installSearch.appendDirectory(Filename.fromOsSpecific(arg))
+        installSearch.appendDirectory(Filename.fromOsSpecific(arg))
     elif opt == '-S':
         tokens = arg.split(',')
         while len(tokens) < 4:
             tokens.append('')
         certificate, chain, pkey, password = tokens[:4]
-        packager.signParams.append((certificate, chain, pkey, password))
+        signParams.append((certificate, chain, pkey, password))
     elif opt == '-D':
-        packager.allowPythonDev = True
+        allowPythonDev = True
+    elif opt == '-u':
+        universalBinaries = True
     elif opt == '-P':
-        packager.platform = arg
+        platforms.append(arg)
         
     elif opt == '-h':
         usage(0)
@@ -140,24 +160,41 @@ packageNames = None
 if len(args) > 1:
     packageNames = args[1:]
 
-if not packager.installDir:
+if not installDir:
     print '\nYou must name the target install directory with the -i parameter.\n'
     sys.exit(1)
-packager.installSearch.prependDirectory(packager.installDir)
 
-try:
-    packager.setup()
-    packages = packager.readPackageDef(packageDef, packageNames = packageNames)
-    packager.close()
-    if buildPatches:
-        packager.buildPatches(packages)
-        
-except Packager.PackagerError:
-    # Just print the error message and exit gracefully.
-    inst = sys.exc_info()[1]
-    print inst.args[0]
-    #raise
-    sys.exit(1)
+if universalBinaries:
+    if platforms:
+        print '\nYou may not specify both -u and -P.\n'
+        sys.exit(1)
+    if PandaSystem.getPlatform().startswith('osx_'):
+        # Maybe soon we'll add osx_x86_64 to this default list.
+        platforms = ['osx_i386', 'osx_ppc']
+
+if not platforms:
+    platforms = [PandaSystem.getPlatform()]
+
+for platform in platforms:
+    packager = Packager.Packager(platform = platform)
+    packager.installDir = installDir
+    packager.installSearch = [installDir] + installSearch + packager.installSearch
+    packager.signParams = signParams
+    packager.allowPythonDev = allowPythonDev
+
+    try:
+        packager.setup()
+        packages = packager.readPackageDef(packageDef, packageNames = packageNames)
+        packager.close()
+        if buildPatches:
+            packager.buildPatches(packages)
+
+    except Packager.PackagerError:
+        # Just print the error message and exit gracefully.
+        inst = sys.exc_info()[1]
+        print inst.args[0]
+        #raise
+        sys.exit(1)
 
 # An explicit call to exit() is required to exit the program, when
 # this module is packaged in a p3d file.

direct/src/showutil/FreezeTool.py

@@ -46,7 +46,9 @@ class CompilationEnvironment:
     can create a custom instance of this class (or simply set the
     compile strings directly) to customize the build environment. """
 
-    def __init__(self):
+    def __init__(self, platform):
+        self.platform = platform
+        
         # The command to compile a c to an object file.  Replace %(basename)s
         # with the basename of the source file, and an implicit .c extension.
         self.compileObj = 'error'
@@ -82,7 +84,7 @@ class CompilationEnvironment:
         self.determineStandardSetup()
 
     def determineStandardSetup(self):
-        if sys.platform == 'win32':
+        if self.platform == 'win32':
             self.Python = PREFIX
 
             if ('VCINSTALLDIR' in os.environ):
@@ -123,10 +125,9 @@ class CompilationEnvironment:
                 self.linkExe = 'link /nologo /MAP:NUL /FIXED:NO /OPT:REF /STACK:4194304 /INCREMENTAL:NO /LIBPATH:"%(PSDK)s\lib" /LIBPATH:"%(MSVC)s\lib" /LIBPATH:"%(python)s\libs"  /out:%(basename)s.exe %(basename)s.obj'
                 self.linkDll = 'link /nologo /DLL /MAP:NUL /FIXED:NO /OPT:REF /INCREMENTAL:NO /LIBPATH:"%(PSDK)s\lib" /LIBPATH:"%(MSVC)s\lib" /LIBPATH:"%(python)s\libs"  /out:%(basename)s%(dllext)s.pyd %(basename)s.obj'
 
-        elif sys.platform == 'darwin':
+        elif self.platform.startswith('osx_'):
             # OSX
-            plat = PandaSystem.getPlatform()
-            proc = plat.split('_', 1)[1]
+            proc = self.platform.split('_', 1)[1]
             if proc == 'i386':
                 self.arch = '-arch i386'
             elif proc == 'ppc':
@@ -535,10 +536,11 @@ class Freezer:
                 args.append('allowChildren = True')
             return 'ModuleDef(%s)' % (', '.join(args))
 
-    def __init__(self, previous = None, debugLevel = 0):
+    def __init__(self, previous = None, debugLevel = 0,
+                 platform = None):
         # Normally, we are freezing for our own platform.  Change this
         # if untrue.
-        self.platform = sys.platform
+        self.platform = platform or PandaSystem.getPlatform()
 
         # This is the compilation environment.  Fill in your own
         # object here if you have custom needs (for instance, for a
@@ -1216,7 +1218,7 @@ class Freezer:
             dllimport = '__declspec(dllimport) '
 
         if not self.cenv:
-            self.cenv = CompilationEnvironment()
+            self.cenv = CompilationEnvironment(platform = self.platform)
             
         if compileToExe:
             code = self.frozenMainCode