Create experimental yapdt (Yet Another Panda3D Deployment Tool)

This uses FreezeTool and a new deploy-stub binary to create frozen
applications without needing a compiler when running the tool. This tool
is for experimenting with deployment options and is currently inflexible
and hacky.

direct/src/showutil/yapdt.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+import marshal
+import os
+import struct
+
+from direct.showutil import FreezeTool
+import panda3d.core as p3d
+
+def make_module_list_entry(code, offset, modulename, module):
+    size = len(code)
+    if getattr(module, "__path__", None):
+        # Indicate package by negative size
+        size = -size
+    return struct.pack('<256sIi', bytes(modulename, 'ascii'), offset, size)
+
+def make_forbidden_module_list_entry(modulename):
+    return struct.pack('<256sIi', bytes(modulename, 'ascii'), 0, 0)
+
+def get_modules(freezer):
+    # Now generate the actual export table.
+    moduleBlob = bytes()
+    codeOffset = 0
+    moduleList = []
+
+    for moduleName, mdef in freezer.getModuleDefs():
+        origName = mdef.moduleName
+        if mdef.forbid:
+            # Explicitly disallow importing this module.
+            moduleList.append(make_forbidden_module_list_entry(moduleName))
+            continue
+
+        assert not mdef.exclude
+        # Allow importing this module.
+        module = freezer.mf.modules.get(origName, None)
+        code = getattr(module, "__code__", None)
+        if code:
+            code = marshal.dumps(code)
+            moduleList.append(make_module_list_entry(code, codeOffset, moduleName, module))
+            moduleBlob += code
+            codeOffset += len(code)
+            continue
+
+        # This is a module with no associated Python code.  It is either
+        # an extension module or a builtin module.  Get the filename, if
+        # it is the former.
+        extensionFilename = getattr(module, '__file__', None)
+
+        if extensionFilename or freezer.linkExtensionModules:
+            freezer.extras.append((moduleName, extensionFilename))
+
+        # If it is a submodule of a frozen module, Python will have
+        # trouble importing it as a builtin module.  Synthesize a frozen
+        # module that loads it as builtin.
+        if '.' in moduleName and freezer.linkExtensionModules:
+            code = compile('import sys;del sys.modules["%s"];import imp;imp.init_builtin("%s")' % (moduleName, moduleName), moduleName, 'exec')
+            code = marshal.dumps(code)
+            moduleList.append(make_module_list_entry(code, codeOffset, moduleName, module))
+            moduleBlob += code
+            codeOffset += len(code)
+        elif '.' in moduleName:
+            # Nothing we can do about this case except warn the user they
+            # are in for some trouble.
+            print('WARNING: Python cannot import extension modules under '
+                  'frozen Python packages; %s will be inaccessible.  '
+                  'passing either -l to link in extension modules or use '
+                  '-x %s to exclude the entire package.' % (moduleName, moduleName.split('.')[0]))
+
+    return moduleBlob, moduleList
+
+
+if __name__ == '__main__':
+    # Setup Freezer
+    freezer = FreezeTool.Freezer()
+    freezer.keepTemporaryFiles = True
+    freezer.addModule('__main__', filename='app.py')
+    freezer.done(addStartupModules=True)
+
+    # Build from pre-built binary stub
+    stub_path = os.path.join(os.path.dirname(p3d.ExecutionEnvironment.get_dtool_name()), '..', 'bin', 'deploy-stub')
+    out_path = 'frozen_app'
+    with open(stub_path, 'rb') as f:
+        stubbin = f.read()
+
+    modblob, modlist = get_modules(freezer)
+
+    with open(out_path, 'wb') as f:
+        f.write(stubbin)
+        listoffset = f.tell()
+        for mod in modlist:
+            f.write(mod)
+        modsoffset = f.tell()
+        f.write(modblob)
+        f.write(struct.pack('<I', listoffset))
+        f.write(struct.pack('<I', modsoffset))
+        f.write(struct.pack('<I', len(modlist)))
+    os.chmod(out_path, 0o755)

makepanda/makepanda.py

@@ -6297,6 +6297,15 @@ if (PkgSkip("CONTRIB")==0 and not RUNTIME):
   TargetAdd('ai.pyd', input=COMMON_PANDA_LIBS)
   TargetAdd('ai.pyd', opts=['PYTHON'])
 
+#
+# DIRECTORY: pandatool/src/deploy-stub
+#
+if True: # TODO
+    OPTS=['DIR:pandatool/src/deploy-stub', 'BUILDING:DEPLOYSTUB', 'PYTHON']
+    TargetAdd('deploy-stub.obj', opts=OPTS, input='deploy-stub.c')
+    TargetAdd('deploy-stub.exe', input='deploy-stub.obj')
+    TargetAdd('deploy-stub.exe', opts=['PYTHON'])
+
 #
 # Generate the models directory and samples directory
 #

pandatool/src/deploy-stub/deploy-stub.c

@@ -0,0 +1,207 @@
+/* Python interpreter main program for frozen scripts */
+
+#include "Python.h"
+#ifdef _WIN32
+#include "malloc.h"
+#endif
+
+#include <stdio.h>
+
+#if PY_MAJOR_VERSION >= 3
+#include <locale.h>
+#endif
+
+#ifdef MS_WINDOWS
+extern void PyWinFreeze_ExeInit(void);
+extern void PyWinFreeze_ExeTerm(void);
+
+extern DL_IMPORT(int) PyImport_ExtendInittab(struct _inittab *newtab);
+#endif
+
+static unsigned char *modblob = NULL;
+
+/* Main program */
+
+int
+Py_FrozenMain(int argc, char **argv)
+{
+    char *p;
+    int n, sts = 1;
+    int inspect = 0;
+    int unbuffered = 0;
+
+#if PY_MAJOR_VERSION >= 3
+    int i;
+    char *oldloc = NULL;
+    wchar_t **argv_copy = NULL;
+    /* We need a second copies, as Python might modify the first one. */
+    wchar_t **argv_copy2 = NULL;
+
+    if (argc > 0) {
+        argv_copy = PyMem_RawMalloc(sizeof(wchar_t*) * argc);
+        argv_copy2 = PyMem_RawMalloc(sizeof(wchar_t*) * argc);
+        if (!argv_copy || !argv_copy2) {
+            fprintf(stderr, "out of memory\n");
+            goto error;
+        }
+    }
+#endif
+
+    Py_FrozenFlag = 1; /* Suppress errors from getpath.c */
+    Py_NoSiteFlag = 1;
+    Py_NoUserSiteDirectory = 1;
+
+    if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0')
+        inspect = 1;
+    if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0')
+        unbuffered = 1;
+
+    if (unbuffered) {
+        setbuf(stdin, (char *)NULL);
+        setbuf(stdout, (char *)NULL);
+        setbuf(stderr, (char *)NULL);
+    }
+
+#if PY_MAJOR_VERSION >= 3
+    oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
+    if (!oldloc) {
+        fprintf(stderr, "out of memory\n");
+        goto error;
+    }
+
+    setlocale(LC_ALL, "");
+    for (i = 0; i < argc; i++) {
+        argv_copy[i] = Py_DecodeLocale(argv[i], NULL);
+        argv_copy2[i] = argv_copy[i];
+        if (!argv_copy[i]) {
+            fprintf(stderr, "Unable to decode the command line argument #%i\n",
+                            i + 1);
+            argc = i;
+            goto error;
+        }
+    }
+    setlocale(LC_ALL, oldloc);
+    PyMem_RawFree(oldloc);
+    oldloc = NULL;
+#endif
+
+#ifdef MS_WINDOWS
+    PyImport_ExtendInittab(extensions);
+#endif /* MS_WINDOWS */
+
+    if (argc >= 1) {
+#if PY_MAJOR_VERSION >= 3
+        Py_SetProgramName(argv_copy[0]);
+#else
+        Py_SetProgramName(argv[0]);
+#endif
+    }
+
+    Py_Initialize();
+#ifdef MS_WINDOWS
+    PyWinFreeze_ExeInit();
+#endif
+
+    if (Py_VerboseFlag)
+        fprintf(stderr, "Python %s\n%s\n",
+            Py_GetVersion(), Py_GetCopyright());
+
+#if PY_MAJOR_VERSION >= 3
+    PySys_SetArgv(argc, argv_copy);
+#else
+    PySys_SetArgv(argc, argv);
+#endif
+
+    n = PyImport_ImportFrozenModule("__main__");
+    if (n == 0)
+        Py_FatalError("__main__ not frozen");
+    if (n < 0) {
+        PyErr_Print();
+        sts = 1;
+    }
+    else
+        sts = 0;
+
+    n = PyImport_ImportFrozenModule("_frozen_importlib");
+    if (n == 0)
+        Py_FatalError("_frozen_importlib not frozen");
+    if (n < 0)
+        PyErr_Print();
+
+    if (inspect && isatty((int)fileno(stdin)))
+        sts = PyRun_AnyFile(stdin, "<stdin>") != 0;
+
+#ifdef MS_WINDOWS
+    PyWinFreeze_ExeTerm();
+#endif
+    Py_Finalize();
+
+#if PY_MAJOR_VERSION >= 3
+error:
+    PyMem_RawFree(argv_copy);
+    if (argv_copy2) {
+        for (i = 0; i < argc; i++)
+            PyMem_RawFree(argv_copy2[i]);
+        PyMem_RawFree(argv_copy2);
+    }
+    PyMem_RawFree(oldloc);
+#endif
+    return sts;
+}
+
+
+int
+main(int argc, char *argv[]) {
+  struct _frozen *_PyImport_FrozenModules;
+  unsigned int listoff, modsoff, fsize, modsize, listsize, nummods;
+  FILE *runtime = fopen(argv[0], "rb");
+
+  // Get offsets
+  fseek(runtime, -12, SEEK_END);
+  fsize = ftell(runtime);
+  fread(&listoff, 4, 1, runtime);
+  fread(&modsoff, 4, 1, runtime);
+  fread(&nummods, 4, 1, runtime);
+  modsize = fsize - modsoff;
+  listsize = modsoff - listoff;
+
+  // Read module blob
+  modblob = malloc(modsize);
+  fseek(runtime, modsoff, SEEK_SET);
+  fread(modblob, modsize, 1, runtime);
+
+  // Read module list
+  _PyImport_FrozenModules = calloc(nummods + 1, sizeof(struct _frozen));
+  fseek(runtime, listoff, SEEK_SET);
+  for (unsigned int modidx = 0; modidx < nummods; ++modidx) {
+    struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
+    char *name = NULL, namebuf[256] = {0};
+    unsigned int nsize, codeptr;
+    int codesize;
+
+    // Name
+    fread(namebuf, 1, 256, runtime);
+    nsize = strlen(namebuf) + 1;
+    name = malloc(nsize);
+    memcpy(name, namebuf, nsize);
+    moddef->name = name;
+
+    // Pointer
+    fread(&codeptr, 4, 1, runtime);
+    moddef->code = modblob + codeptr;
+
+    // Size
+    fread(&codesize, 4, 1, runtime);
+    moddef->size = codesize;
+  }
+
+  // Uncomment this to print out the read in module list
+  //for (unsigned int modidx = 0; modidx < nummods + 1; ++modidx) {
+  //  struct _frozen *moddef = &_PyImport_FrozenModules[modidx];
+  //  printf("MOD: %s %p %d\n", moddef->name, (void*)moddef->code, moddef->size);
+  //}
+  fclose(runtime);
+
+  PyImport_FrozenModules = _PyImport_FrozenModules;
+  return Py_FrozenMain(argc, argv);
+}