android: add activity for running Python programs

It can be launched from the termux shell using the provided run_python.sh script, which can communicate with the Panda activity using a socket (which is the only way we can reliably get command-line output back to the program.)

The Python script needs to be readable by the Panda activity (which implies it needs to be in /sdcard).

The standard library is packed into the .apk, and loaded using zipimport.  Extension modules are included using a special naming convention and import hook in order to comply with Android's strict demands on how libraries must be named to be included in an .apk.

[skip ci]

makepanda/makepanda.py

@@ -5082,7 +5082,7 @@ if (PkgSkip("SPEEDTREE")==0):
 # DIRECTORY: panda/src/testbed/
 #
 
-if (not RTDIST and not RUNTIME and PkgSkip("PVIEW")==0 and GetTarget() != 'android'):
+if (not RTDIST and not RUNTIME and PkgSkip("PVIEW")==0):
   OPTS=['DIR:panda/src/testbed']
   TargetAdd('pview_pview.obj', opts=OPTS, input='pview.cxx')
   TargetAdd('pview.exe', input='pview_pview.obj')
@@ -5101,6 +5101,7 @@ if (not RUNTIME and GetTarget() == 'android'):
   TargetAdd('org/panda3d/android/NativeIStream.class', opts=OPTS, input='NativeIStream.java')
   TargetAdd('org/panda3d/android/NativeOStream.class', opts=OPTS, input='NativeOStream.java')
   TargetAdd('org/panda3d/android/PandaActivity.class', opts=OPTS, input='PandaActivity.java')
+  TargetAdd('org/panda3d/android/PythonActivity.class', opts=OPTS, input='PythonActivity.java')
 
   TargetAdd('p3android_composite1.obj', opts=OPTS, input='p3android_composite1.cxx')
   TargetAdd('libp3android.dll', input='p3android_composite1.obj')
@@ -5111,10 +5112,10 @@ if (not RUNTIME and GetTarget() == 'android'):
   TargetAdd('android_main.obj', opts=OPTS, input='android_main.cxx')
 
   if (not RTDIST and PkgSkip("PVIEW")==0):
-    TargetAdd('pview_pview.obj', opts=OPTS, input='pview.cxx')
+    TargetAdd('libpview_pview.obj', opts=OPTS, input='pview.cxx')
     TargetAdd('libpview.dll', input='android_native_app_glue.obj')
     TargetAdd('libpview.dll', input='android_main.obj')
-    TargetAdd('libpview.dll', input='pview_pview.obj')
+    TargetAdd('libpview.dll', input='libpview_pview.obj')
     TargetAdd('libpview.dll', input='libp3framework.dll')
     if not PkgSkip("EGG"):
       TargetAdd('libpview.dll', input='libpandaegg.dll')
@@ -5122,6 +5123,17 @@ if (not RUNTIME and GetTarget() == 'android'):
     TargetAdd('libpview.dll', input=COMMON_PANDA_LIBS)
     TargetAdd('libpview.dll', opts=['MODULE', 'ANDROID'])
 
+  if (not RTDIST and PkgSkip("PYTHON")==0):
+    OPTS += ['PYTHON']
+    TargetAdd('ppython_ppython.obj', opts=OPTS, input='python_main.cxx')
+    TargetAdd('libppython.dll', input='android_native_app_glue.obj')
+    TargetAdd('libppython.dll', input='android_main.obj')
+    TargetAdd('libppython.dll', input='ppython_ppython.obj')
+    TargetAdd('libppython.dll', input='libp3framework.dll')
+    TargetAdd('libppython.dll', input='libp3android.dll')
+    TargetAdd('libppython.dll', input=COMMON_PANDA_LIBS)
+    TargetAdd('libppython.dll', opts=['MODULE', 'ANDROID', 'PYTHON'])
+
 #
 # DIRECTORY: panda/src/androiddisplay/
 #
@@ -7505,7 +7517,7 @@ def MakeInstallerAndroid():
                 continue
             if '.so.' in line:
                 dep = line.rpartition('.so.')[0] + '.so'
-                oscmd("patchelf --replace-needed %s %s %s" % (line, dep, target))
+                oscmd("patchelf --replace-needed %s %s %s" % (line, dep, target), True)
             else:
                 dep = line
 
@@ -7516,6 +7528,7 @@ def MakeInstallerAndroid():
                     copy_library(os.path.realpath(fulldep), dep)
                     break
 
+    # Now copy every lib in the lib dir, and its dependencies.
     for base in os.listdir(source_dir):
         if not base.startswith('lib'):
             continue
@@ -7527,6 +7540,59 @@ def MakeInstallerAndroid():
             continue
         copy_library(source, base)
 
+    # Same for Python extension modules.  However, Android is strict about
+    # library naming, so we have a special naming scheme for these, in
+    # conjunction with a custom import hook to find these modules.
+    if not PkgSkip("PYTHON"):
+        suffix = GetExtensionSuffix()
+        source_dir = os.path.join(GetOutputDir(), "panda3d")
+        for base in os.listdir(source_dir):
+            if not base.endswith(suffix):
+                continue
+            modname = base[:-len(suffix)]
+            source = os.path.join(source_dir, base)
+            copy_library(source, "libpy.panda3d.{}.so".format(modname))
+
+        # Same for standard Python modules.
+        import _ctypes
+        source_dir = os.path.dirname(_ctypes.__file__)
+        for base in os.listdir(source_dir):
+            if not base.endswith('.so'):
+                continue
+            modname = base.partition('.')[0]
+            source = os.path.join(source_dir, base)
+            copy_library(source, "libpy.{}.so".format(modname))
+
+    def copy_python_tree(source_root, target_root):
+        for source_dir, dirs, files in os.walk(source_root):
+            if 'site-packages' in dirs:
+                dirs.remove('site-packages')
+
+            if not any(base.endswith('.py') for base in files):
+                continue
+
+            target_dir = os.path.join(target_root, os.path.relpath(source_dir, source_root))
+            target_dir = os.path.normpath(target_dir)
+            os.makedirs(target_dir, 0o755)
+
+            for base in files:
+                if base.endswith('.py'):
+                    target = os.path.join(target_dir, base)
+                    shutil.copy(os.path.join(source_dir, base), target)
+
+    # Copy the Python standard library to the .apk as well.
+    from distutils.sysconfig import get_python_lib
+    stdlib_source = get_python_lib(False, True)
+    stdlib_target = os.path.join("apkroot", "lib", "python{0}.{1}".format(*sys.version_info))
+    copy_python_tree(stdlib_source, stdlib_target)
+
+    # But also copy over our custom site.py.
+    shutil.copy("panda/src/android/site.py", os.path.join(stdlib_target, "site.py"))
+
+    # And now make a site-packages directory containing our direct/panda3d/pandac modules.
+    for tree in "panda3d", "direct", "pandac":
+        copy_python_tree(os.path.join(GetOutputDir(), tree), os.path.join(stdlib_target, "site-packages", tree))
+
     # Copy the models and config files to the virtual assets filesystem.
     oscmd("mkdir apkroot/assets")
     oscmd("cp -R %s apkroot/assets/models" % (os.path.join(GetOutputDir(), "models")))
@@ -7545,7 +7611,11 @@ def MakeInstallerAndroid():
     oscmd(aapt_cmd)
 
     # And add all the libraries to it.
-    oscmd("cd apkroot && aapt add ../%s classes.dex lib/%s/lib*.so" % (apk_unaligned, SDK["ANDROID_ABI"]))
+    oscmd("cd apkroot && aapt add ../%s classes.dex" % (apk_unaligned))
+    for path, dirs, files in os.walk('apkroot/lib'):
+        if files:
+            rel = os.path.relpath(path, 'apkroot')
+            oscmd("cd apkroot && aapt add ../%s %s/*" % (apk_unaligned, rel))
 
     # Now align the .apk, which is necessary for Android to load it.
     oscmd("zipalign -v -p 4 %s %s" % (apk_unaligned, apk_unsigned))

panda/src/android/PythonActivity.java

@@ -0,0 +1,23 @@
+/**
+ * PANDA 3D SOFTWARE
+ * Copyright (c) Carnegie Mellon University.  All rights reserved.
+ *
+ * All use of this software is subject to the terms of the revised BSD
+ * license.  You should have received a copy of this license along
+ * with this source code in a file named "LICENSE."
+ *
+ * @file PythonActivity.java
+ * @author rdb
+ * @date 2018-02-04
+ */
+
+package org.panda3d.android;
+
+import org.panda3d.android.PandaActivity;
+
+/**
+ * This is only declared as a separate class from PandaActivity so that we
+ * can have two separate activity definitions in ApplicationManifest.xml.
+ */
+public class PythonActivity extends PandaActivity {
+}

panda/src/android/pview_manifest.xml

@@ -45,13 +45,38 @@
             <intent-filter>
                 <action android:name="android.intent.action.VIEW" />
                 <category android:name="android.intent.category.DEFAULT" />
-                <data android:mimeType="*/*" scheme="content" host="com.termux.files" />
-                <data android:pathPattern=".*\\.egg" />
-                <data android:pathPattern=".*\\.egg.pz" />
-                <data android:pathPattern=".*\\.egg.gz" />
-                <data android:pathPattern=".*\\.bam" />
-                <data android:pathPattern=".*\\.bam.pz" />
-                <data android:pathPattern=".*\\.bam.gz" />
+                <data android:mimeType="*/*" android:scheme="content" android:host="com.termux.files" android:pathPattern=".*\\.egg" />
+            </intent-filter>
+        </activity>
+        <activity android:name="org.panda3d.android.PythonActivity"
+                android:label="Panda Python" android:theme="@android:style/Theme.NoTitleBar"
+                android:configChanges="orientation|keyboardHidden"
+                android:launchMode="singleInstance">
+
+            <meta-data android:name="android.app.lib_name"
+                       android:value="ppython" />
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+            <intent-filter>
+                <action android:name="android.intent.action.VIEW" />
+                <category android:name="android.intent.category.DEFAULT" />
+                <data android:mimeType="text/x-python" />
+            </intent-filter>
+            <intent-filter>
+                <action android:name="android.intent.action.VIEW" />
+                <category android:name="android.intent.category.DEFAULT" />
+                <data android:mimeType="*/*" android:scheme="file" />
+                <data android:pathPattern=".*\\.py" />
+                <data android:pathPattern=".*\\.pyw" />
+            </intent-filter>
+            <intent-filter>
+                <action android:name="android.intent.action.VIEW" />
+                <category android:name="android.intent.category.DEFAULT" />
+                <data android:mimeType="*/*" android:scheme="content" android:host="com.termux.files" />
+                <data android:pathPattern=".*\\.py" />
+                <data android:pathPattern=".*\\.pyw" />
             </intent-filter>
         </activity>
     </application>

panda/src/android/python_main.cxx

@@ -0,0 +1,80 @@
+/**
+ * PANDA 3D SOFTWARE
+ * Copyright (c) Carnegie Mellon University.  All rights reserved.
+ *
+ * All use of this software is subject to the terms of the revised BSD
+ * license.  You should have received a copy of this license along
+ * with this source code in a file named "LICENSE."
+ *
+ * @file python_main.cxx
+ * @author rdb
+ * @date 2018-02-12
+ */
+
+#include "dtoolbase.h"
+#include "config_android.h"
+#include "executionEnvironment.h"
+
+#undef _POSIX_C_SOURCE
+#undef _XOPEN_SOURCE
+#include <Python.h>
+#if PY_MAJOR_VERSION >= 3
+#include <wchar.h>
+#endif
+
+#include <dlfcn.h>
+
+/**
+ * The main entry point for the Python activity.  Called by android_main.
+ */
+int main(int argc, char *argv[]) {
+  if (argc <= 1) {
+    return 1;
+  }
+
+  // Help out Python by telling it which encoding to use
+  Py_FileSystemDefaultEncoding = "utf-8";
+
+  Py_SetProgramName(Py_DecodeLocale("ppython", nullptr));
+
+  // Set PYTHONHOME to the location of the .apk file.
+  string apk_path = ExecutionEnvironment::get_binary_name();
+  Py_SetPythonHome(Py_DecodeLocale(apk_path.c_str(), nullptr));
+
+  // We need to make zlib available to zipimport, but I don't know how
+  // we could inject our import hook before Py_Initialize, so instead
+  // load it as though it were a built-in module.
+  void *zlib = dlopen("libpy.zlib.so", RTLD_NOW);
+  if (zlib != nullptr) {
+    void *init = dlsym(zlib, "PyInit_zlib");
+    if (init != nullptr) {
+      PyImport_AppendInittab("zlib", (PyObject *(*)())init);
+    }
+  }
+
+  Py_Initialize();
+
+  // This is used by the import hook to locate the module libraries.
+  Filename dtool_name = ExecutionEnvironment::get_dtool_name();
+  string native_dir = dtool_name.get_dirname();
+  PyObject *py_native_dir = PyUnicode_FromStringAndSize(native_dir.c_str(), native_dir.size());
+  PySys_SetObject("_native_library_dir", py_native_dir);
+  Py_DECREF(py_native_dir);
+
+  int sts = 1;
+  FILE *fp = fopen(argv[1], "r");
+  if (fp != nullptr) {
+    int res = PyRun_AnyFile(fp, argv[1]);
+    if (res > 0) {
+      sts = 0;
+    } else {
+      android_cat.error() << "Error running " << argv[1] << "\n";
+      PyErr_Print();
+    }
+  } else {
+    android_cat.error() << "Unable to open " << argv[1] << "\n";
+  }
+
+  Py_Finalize();
+  return sts;
+}

panda/src/android/run_pview.sh

@@ -0,0 +1,14 @@
+# This script can be used for launching the Panda viewer from the Android
+# terminal environment, for example from within termux.  It uses a socket
+# to pipe the command-line output back to the terminal.
+
+port=12345
+
+if [[ $# -eq 0 ]] ; then
+    echo "Pass full path of model"
+    exit 1
+fi
+
+am start --activity-clear-task -n org.panda3d.sdk/org.panda3d.android.PandaActivity --user 0 --es org.panda3d.OUTPUT_URI tcp://127.0.0.1:$port --grant-read-uri-permission --grant-write-uri-permission file://$(realpath $1)
+
+nc -l -p $port

panda/src/android/run_python.sh

@@ -0,0 +1,14 @@
+# This script can be used for launching a Python script from the Android
+# terminal environment, for example from within termux.  It uses a socket
+# to pipe the command-line output back to the terminal.
+
+port=12345
+
+if [[ $# -eq 0 ]] ; then
+    echo "Pass full path of script"
+    exit 1
+fi
+
+am start --activity-clear-task -n org.panda3d.sdk/org.panda3d.android.PythonActivity --user 0 --es org.panda3d.OUTPUT_URI tcp://127.0.0.1:$port --grant-read-uri-permission --grant-write-uri-permission file://$(realpath $1)
+
+nc -l -p $port

panda/src/android/site.py

@@ -0,0 +1,34 @@
+import sys
+import os
+
+from importlib.abc import Loader, MetaPathFinder
+from importlib.machinery import ModuleSpec
+
+if sys.version_info >= (3, 5):
+    from importlib import _bootstrap_external
+else:
+    from importlib import _bootstrap as _bootstrap_external
+
+sys.platform = "android"
+
+class AndroidExtensionFinder(MetaPathFinder):
+    @classmethod
+    def find_spec(cls, fullname, path=None, target=None):
+        soname = 'libpy.' + fullname + '.so'
+        path = os.path.join(sys._native_library_dir, soname)
+
+        if os.path.exists(path):
+            loader = _bootstrap_external.ExtensionFileLoader(fullname, path)
+            return ModuleSpec(fullname, loader, origin=path)
+
+
+def main():
+    """Adds the site-packages directory to the sys.path.
+    Also, registers the import hook for extension modules."""
+
+    sys.path.append('{0}/lib/python{1}.{2}/site-packages'.format(sys.prefix, *sys.version_info))
+    sys.meta_path.append(AndroidExtensionFinder)
+
+
+if not sys.flags.no_site:
+    main()