makepanda: allow building multiple Python versions in one built dir

This is done by adding a PyTargetAdd function, which builds the target into a Python ABI-specific temporary directory, allowing multiple Python versions to be built into the same built dir side-by-side.  This could greatly speed up buildbot builds.

It also paves the way for building multiple Python versions in the same makepanda call / installer by changing PyTargetAdd to add one target per enabled Python version.

makepanda/installer.nsi

@@ -331,13 +331,24 @@ SectionGroup "Python support"
         SetOutPath $INSTDIR\panda3d
         File /r "${BUILT}\panda3d\*.py"
 
-        File /r /x bullet.pyd /x ode.pyd /x physx.pyd /x rocket.pyd "${BUILT}\panda3d\*.pyd"
+        File /nonfatal /r "${BUILT}\panda3d\core${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\ai${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\awesomium${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\direct${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\egg${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\fx${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\interrogatedb${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\physics${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\_rplight${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\skel${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\vision${EXT_SUFFIX}"
+        File /nonfatal /r "${BUILT}\panda3d\vrpn${EXT_SUFFIX}"
 
         !ifdef HAVE_BULLET
             SectionGetFlags ${SecBullet} $R0
             IntOp $R0 $R0 & ${SF_SELECTED}
             StrCmp $R0 ${SF_SELECTED} 0 SkipBulletPyd
-            File /nonfatal /r "${BUILT}\panda3d\bullet.pyd"
+            File /nonfatal /r "${BUILT}\panda3d\bullet${EXT_SUFFIX}"
             SkipBulletPyd:
         !endif
 
@@ -345,7 +356,7 @@ SectionGroup "Python support"
             SectionGetFlags ${SecODE} $R0
             IntOp $R0 $R0 & ${SF_SELECTED}
             StrCmp $R0 ${SF_SELECTED} 0 SkipODEPyd
-            File /nonfatal /r "${BUILT}\panda3d\ode.pyd"
+            File /nonfatal /r "${BUILT}\panda3d\ode${EXT_SUFFIX}"
             SkipODEPyd:
         !endif
 
@@ -353,7 +364,7 @@ SectionGroup "Python support"
             SectionGetFlags ${SecPhysX} $R0
             IntOp $R0 $R0 & ${SF_SELECTED}
             StrCmp $R0 ${SF_SELECTED} 0 SkipPhysXPyd
-            File /nonfatal /r "${BUILT}\panda3d\physx.pyd"
+            File /nonfatal /r "${BUILT}\panda3d\physx${EXT_SUFFIX}"
             SkipPhysXPyd:
         !endif
 
@@ -361,7 +372,7 @@ SectionGroup "Python support"
             SectionGetFlags ${SecRocket} $R0
             IntOp $R0 $R0 & ${SF_SELECTED}
             StrCmp $R0 ${SF_SELECTED} 0 SkipRocketPyd
-            File /nonfatal /r "${BUILT}\panda3d\rocket.pyd"
+            File /nonfatal /r "${BUILT}\panda3d\rocket${EXT_SUFFIX}"
             SkipRocketPyd:
         !endif
 

makepanda/installpanda.py

@@ -175,6 +175,7 @@ def InstallPanda(destdir="", prefix="/usr", outputdir="built", libdir=GetLibDir(
     oscmd("mkdir -m 0755 -p "+destdir+prefix+"/share/applications")
     oscmd("mkdir -m 0755 -p "+destdir+libdir+"/panda3d")
     oscmd("mkdir -m 0755 -p "+destdir+PPATH)
+    oscmd("mkdir -m 0755 -p "+destdir+PPATH+"/panda3d")
 
     if (sys.platform.startswith("freebsd")):
         oscmd("mkdir -m 0755 -p "+destdir+prefix+"/etc")
@@ -194,13 +195,17 @@ def InstallPanda(destdir="", prefix="/usr", outputdir="built", libdir=GetLibDir(
 
     oscmd("cp -R "+outputdir+"/include          "+destdir+prefix+"/include/panda3d")
     oscmd("cp -R "+outputdir+"/pandac           "+destdir+prefix+"/share/panda3d/")
-    oscmd("cp -R "+outputdir+"/panda3d          "+destdir+PPATH+"/")
     oscmd("cp -R "+outputdir+"/models           "+destdir+prefix+"/share/panda3d/")
     if os.path.isdir("samples"):             oscmd("cp -R samples               "+destdir+prefix+"/share/panda3d/")
     if os.path.isdir(outputdir+"/direct"):   oscmd("cp -R "+outputdir+"/direct           "+destdir+prefix+"/share/panda3d/")
     if os.path.isdir(outputdir+"/Pmw"):      oscmd("cp -R "+outputdir+"/Pmw     "+destdir+prefix+"/share/panda3d/")
     if os.path.isdir(outputdir+"/plugins"):  oscmd("cp -R "+outputdir+"/plugins "+destdir+prefix+"/share/panda3d/")
 
+    suffix = GetExtensionSuffix()
+    for base in os.listdir(outputdir + "/panda3d"):
+        if base.endswith(".py") or (base.endswith(suffix) and '.' not in base[:-len(suffix)]):
+            oscmd("cp "+outputdir+"/panda3d/"+base+" "+destdir+PPATH+"/panda3d/"+base)
+
     WriteMimeFile(destdir+prefix+"/share/mime-info/panda3d.mime", MIME_INFO)
     WriteKeysFile(destdir+prefix+"/share/mime-info/panda3d.keys", MIME_INFO)
     WriteMimeXMLFile(destdir+prefix+"/share/mime/packages/panda3d.xml", MIME_INFO)

makepanda/makepandacore.py

@@ -3239,6 +3239,8 @@ def WriteEmbeddedStringFile(basename, inputs, string_name=None):
 ########################################################################
 
 ORIG_EXT = {}
+PYABI_SPECIFIC = set()
+WARNED_FILES = set()
 
 def GetOrigExt(x):
     return ORIG_EXT[x]
@@ -3249,14 +3251,42 @@ def SetOrigExt(x, v):
 def GetExtensionSuffix():
     if sys.version_info >= (3, 0):
         suffix = sysconfig.get_config_var('EXT_SUFFIX')
-        if suffix:
+        if suffix == '.so':
+            # On my FreeBSD system, this is not set correctly, but SOABI is.
+            soabi = sysconfig.get_config_var('SOABI')
+            if soabi:
+                return '.%s.so' % (soabi)
+        elif suffix:
             return suffix
+
     target = GetTarget()
     if target == 'windows':
         return '.pyd'
     else:
         return '.so'
 
+def GetPythonABI():
+    soabi = sysconfig.get_config_var('SOABI')
+    if soabi:
+        return soabi
+
+    soabi = 'cpython-%d%d' % (sys.version_info[:2])
+
+    debug_flag = sysconfig.get_config_var('Py_DEBUG')
+    if (debug_flag is None and hasattr(sys, 'gettotalrefcount')) or debug_flag:
+        soabi += 'd'
+
+    malloc_flag = sysconfig.get_config_var('WITH_PYMALLOC')
+    if malloc_flag is None or malloc_flag:
+        soabi += 'm'
+
+    if sys.version_info < (3, 3):
+        usize = sysconfig.get_config_var('Py_UNICODE_SIZE')
+        if (usize is None and sys.maxunicode == 0x10ffff) or usize == 4:
+            soabi += 'u'
+
+    return soabi
+
 def CalcLocation(fn, ipath):
     if fn.startswith("panda3d/") and fn.endswith(".py"):
         return OUTPUTDIR + "/" + fn
@@ -3327,11 +3357,25 @@ def CalcLocation(fn, ipath):
     return fn
 
 
-def FindLocation(fn, ipath):
+def FindLocation(fn, ipath, pyabi=None):
     if (GetLinkAllStatic() and fn.endswith(".dll")):
         fn = fn[:-4] + ".lib"
     loc = CalcLocation(fn, ipath)
     base, ext = os.path.splitext(fn)
+
+    # If this is a target created with PyTargetAdd, we need to make sure it
+    # it put in a Python-version-specific directory.
+    if loc in PYABI_SPECIFIC:
+        if loc.startswith(OUTPUTDIR + "/tmp"):
+            if pyabi is not None:
+                loc = OUTPUTDIR + "/tmp/" + pyabi + loc[len(OUTPUTDIR) + 4:]
+            else:
+                raise RuntimeError("%s is a Python-specific target, use PyTargetAdd instead of TargetAdd" % (fn))
+
+        elif ext != ".pyd" and loc not in WARNED_FILES:
+            WARNED_FILES.add(loc)
+            print("%sWARNING:%s file depends on Python but is not in an ABI-specific directory: %s%s%s" % (GetColor("red"), GetColor(), GetColor("green"), loc, GetColor()))
+
     ORIG_EXT[loc] = ext
     return loc
 
@@ -3377,6 +3421,11 @@ def FindLocation(fn, ipath):
 ## be inserted: bison generates an OBJ and a secondary header
 ## file, interrogate generates an IN and a secondary IGATE.OBJ.
 ##
+## PyTargetAdd is a special version for targets that depend on Python.
+## It will create a target for each Python version we are building with,
+## ensuring that builds with different Python versions won't conflict
+## when we build for multiple Python ABIs side-by-side.
+##
 ########################################################################
 
 class Target:
@@ -3385,7 +3434,7 @@ class Target:
 TARGET_LIST = []
 TARGET_TABLE = {}
 
-def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None):
+def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None, pyabi=None):
     if (dummy != 0):
         exit("Syntax error in TargetAdd "+target)
     if ipath is None: ipath = opts
@@ -3393,11 +3442,10 @@ def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None
     if (type(input) == str): input = [input]
     if (type(dep) == str): dep = [dep]
 
-    if os.path.splitext(target)[1] == '.pyd' and PkgSkip("PYTHON"):
-        # It makes no sense to build Python modules with python disabled.
-        return
+    if target.endswith(".pyd") and not pyabi:
+        raise RuntimeError("Use PyTargetAdd to build .pyd targets")
 
-    full = FindLocation(target, [OUTPUTDIR + "/include"])
+    full = FindLocation(target, [OUTPUTDIR + "/include"], pyabi=pyabi)
 
     if (full not in TARGET_TABLE):
         t = Target()
@@ -3416,7 +3464,7 @@ def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None
 
     ipath = [OUTPUTDIR + "/tmp"] + GetListOption(ipath, "DIR:") + [OUTPUTDIR+"/include"]
     for x in input:
-        fullinput = FindLocation(x, ipath)
+        fullinput = FindLocation(x, ipath, pyabi=pyabi)
         t.inputs.append(fullinput)
         # Don't re-link a library or binary if just its dependency dlls have been altered.
         # This should work out fine in most cases, and often reduces recompilation time.
@@ -3455,7 +3503,7 @@ def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None
                 t.deps[fulln] = 1
 
     for x in dep:
-        fulldep = FindLocation(x, ipath)
+        fulldep = FindLocation(x, ipath, pyabi=pyabi)
         t.deps[fulldep] = 1
 
     if winrc and GetTarget() == 'windows':
@@ -3472,3 +3520,32 @@ def TargetAdd(target, dummy=0, opts=[], input=[], dep=[], ipath=None, winrc=None
 
     if target.endswith(".pz") and not CrossCompiling():
         t.deps[FindLocation("pzip.exe", [])] = 1
+
+    if target.endswith(".in"):
+        # Also add a target to compile the _igate.cxx file into an _igate.obj.
+        outbase = os.path.basename(target)[:-3]
+        woutc = OUTPUTDIR + "/tmp/" + outbase + "_igate.cxx"
+        CxxDependencyCache[woutc] = []
+        PyTargetAdd(outbase + "_igate.obj", opts=opts+['PYTHON','BIGOBJ'], input=woutc, dep=target)
+
+
+def PyTargetAdd(target, opts=[], **kwargs):
+    if PkgSkip("PYTHON"):
+        return
+
+    if 'PYTHON' not in opts:
+        opts = opts + ['PYTHON']
+
+    abi = GetPythonABI()
+
+    MakeDirectory(OUTPUTDIR + "/tmp/" + abi)
+
+    # Mark this target as being a Python-specific target.
+    orig = CalcLocation(target, [OUTPUTDIR + "/include"])
+    PYABI_SPECIFIC.add(orig)
+
+    if orig.startswith(OUTPUTDIR + "/tmp/") and os.path.exists(orig):
+        print("Removing file %s" % (orig))
+        os.unlink(orig)
+
+    TargetAdd(target, opts=opts, pyabi=abi, **kwargs)

makepanda/makewheel.py

@@ -498,7 +498,7 @@ __version__ = '{0}'
     for file in os.listdir(panda3d_dir):
         if file == '__init__.py':
             pass
-        elif file.endswith(ext_suffix) or file.endswith('.py'):
+        elif file.endswith('.py') or (file.endswith(ext_suffix) and '.' not in file[:-len(ext_suffix)]):
             source_path = os.path.join(panda3d_dir, file)
 
             if file.endswith('.pyd') and platform.startswith('cygwin'):