Work towards supporting Python 3 in the rtdist

direct/src/p3d/Packager.py

@@ -467,7 +467,7 @@ class Packager:
             if self.p3dApplication:
                 allowPythonDev = self.configs.get('allow_python_dev', 0)
                 if int(allowPythonDev):
-                    print "\n*** Generating %s.p3d with allow_python_dev enabled ***\n" % (self.packageName)
+                    print("\n*** Generating %s.p3d with allow_python_dev enabled ***\n" % (self.packageName))
 
             return result
 
@@ -721,7 +721,7 @@ class Packager:
             self.packageDesc = packageDir + self.packageDesc
             self.packageImportDesc = packageDir + self.packageImportDesc
 
-            print "Generating %s" % (self.packageFilename)
+            print("Generating %s" % (self.packageFilename))
 
             if self.p3dApplication:
                 self.packageFullpath = Filename(self.packager.p3dInstallDir, self.packageFilename)
@@ -1251,13 +1251,13 @@ class Packager:
                 elf.close()
                 return None
 
-            if not ident.startswith("\177ELF"):
+            if not ident.startswith(b"\177ELF"):
                 # No elf magic!  Beware of orcs.
                 return None
 
             # Make sure we read in the correct endianness and integer size
-            byteOrder = "<>"[ord(ident[5]) - 1]
-            elfClass = ord(ident[4]) - 1 # 0 = 32-bits, 1 = 64-bits
+            byteOrder = "<>"[ord(ident[5:6]) - 1]
+            elfClass = ord(ident[4:5]) - 1 # 0 = 32-bits, 1 = 64-bits
             headerStruct = byteOrder + ("HHIIIIIHHHHHH", "HHIQQQIHHHHHH")[elfClass]
             sectionStruct = byteOrder + ("4xI8xIII8xI", "4xI16xQQI12xQ")[elfClass]
             dynamicStruct = byteOrder + ("iI", "qQ")[elfClass]
@@ -1291,17 +1291,17 @@ class Packager:
                 elf.seek(offset)
                 data = elf.read(entsize)
                 tag, val = struct.unpack_from(dynamicStruct, data)
-                newSectionData = ""
+                newSectionData = b""
                 startReplace = None
                 pad = 0
 
                 # Read tags until we find a NULL tag.
                 while tag != 0:
                     if tag == 1: # A NEEDED entry.  Read it from the string table.
-                        filenames.append(stringTables[link][val : stringTables[link].find('\0', val)])
+                        filenames.append(stringTables[link][val : stringTables[link].find(b'\0', val)])
 
                     elif tag == 15 or tag == 29:
-                        rpath += stringTables[link][val : stringTables[link].find('\0', val)].split(':')
+                        rpath += stringTables[link][val : stringTables[link].find(b'\0', val)].split(b':')
                         # An RPATH or RUNPATH entry.
                         if not startReplace:
                             startReplace = elf.tell() - entsize
@@ -1315,7 +1315,7 @@ class Packager:
                     tag, val = struct.unpack_from(dynamicStruct, data)
 
                 if startReplace is not None:
-                    newSectionData += data + ("\0" * pad)
+                    newSectionData += data + (b"\0" * pad)
                     rewriteSections.append((startReplace, newSectionData))
             elf.close()
 
@@ -1348,7 +1348,7 @@ class Packager:
             for offset, data in rewriteSections:
                 elf.seek(offset)
                 elf.write(data)
-            elf.write("\0" * pad)
+            elf.write(b"\0" * pad)
             elf.close()
             return filenames
 
@@ -2486,7 +2486,7 @@ class Packager:
         if not os.path.isfile('/sbin/ldconfig'):
             return False
 
-        handle = subprocess.Popen(['/sbin/ldconfig', '-p'], stdout=subprocess.PIPE)
+        handle = subprocess.Popen(['/sbin/ldconfig', '-p'], stdout=subprocess.PIPE, universal_newlines=True)
         out, err = handle.communicate()
 
         if handle.returncode != 0:
@@ -3069,7 +3069,7 @@ class Packager:
         tuples = []
         for package in packages:
             version = self.__makeVersionTuple(package.version)
-            tuples.append((version, file))
+            tuples.append((version, package))
         tuples.sort(reverse = True)
 
         return [t[1] for t in tuples]
@@ -3082,7 +3082,7 @@ class Packager:
         tuples = []
         for package in packages:
             version = self.__makeVersionTuple(package.packageVersion)
-            tuples.append((version, file))
+            tuples.append((version, package))
         tuples.sort(reverse = True)
 
         return [t[1] for t in tuples]
@@ -3141,9 +3141,9 @@ class Packager:
         if panda1.version == panda2.version:
             return True
 
-        print 'Rejecting package %s, version "%s": depends on %s, version "%s" instead of version "%s"' % (
+        print('Rejecting package %s, version "%s": depends on %s, version "%s" instead of version "%s"' % (
             package.packageName, package.version,
-            panda1.packageName, panda1.version, panda2.version)
+            panda1.packageName, panda1.version, panda2.version))
         return False
 
     def __findPackageInRequires(self, packageName, list):

direct/src/p3d/coreapi.pdef

@@ -66,7 +66,7 @@ class images(package):
             token = '%s_img' % (name)
             configDict[token] = basename
         else:
-            print "Could not locate %s" % (filename)
+            print("Could not locate %s" % (filename))
 
     # Also make a few special cases.  We use the same default image
     # for many of the states.

direct/src/p3d/ppackage.py

@@ -139,12 +139,12 @@ from direct.p3d import Packager
 from panda3d.core import *
 
 def usage(code, msg = ''):
-    print >> sys.stderr, usageText % {
+    sys.stderr.write(usageText % {
         'version' : PandaSystem.getPackageVersionString(),
         'host' : PandaSystem.getPackageHostUrl(),
         'prog' : os.path.split(sys.argv[0])[1]
-        }
-    print >> sys.stderr, msg
+        })
+    sys.stderr.write(msg + '\n')
     sys.exit(code)
 
 installDir = None
@@ -161,7 +161,7 @@ platforms = []
 
 try:
     opts, args = getopt.getopt(sys.argv[1:], 'i:ps:S:DuP:R:Ha:hv')
-except getopt.error, msg:
+except getopt.error as msg:
     usage(1, msg)
 
 for opt, arg in opts:
@@ -199,7 +199,7 @@ for opt, arg in opts:
     elif opt == '-h':
         usage(0)
     else:
-        print 'illegal option: ' + arg
+        print('illegal option: ' + arg)
         sys.exit(1)
 
 if not args:
@@ -220,7 +220,7 @@ else:
 
 if universalBinaries:
     if platforms:
-        print '\nYou may not specify both -u and -P.\n'
+        print('\nYou may not specify both -u and -P.\n')
         sys.exit(1)
     if PandaSystem.getPlatform().startswith('osx_'):
         platforms = ['osx_i386', 'osx_amd64']
@@ -251,7 +251,7 @@ for platform in platforms:
     except Packager.PackagerError:
         # Just print the error message and exit gracefully.
         inst = sys.exc_info()[1]
-        print inst.args[0]
+        print(inst.args[0])
         sys.exit(1)
 
 # An explicit call to exit() is required to exit the program, when

direct/src/plugin/p3dPythonRun.cxx

@@ -54,16 +54,26 @@ P3DPythonRun(const char *program_name, const char *archive_file,
   _interactive_console = interactive_console;
 
   if (program_name != NULL) {
+#if PY_MAJOR_VERSION >= 3
+    // Python 3 case: we have to convert it to a wstring.
+    TextEncoder enc;
+    enc.set_text(program_name);
+    _program_name = enc.get_wtext();
+#else
+    // Python 2 is happy with a regular string.
     _program_name = program_name;
+#endif
   }
   if (archive_file != NULL) {
     _archive_file = Filename::from_os_specific(archive_file);
   }
 
   _py_argc = 1;
-  _py_argv = (char **)malloc(2 * sizeof(char *));
-
+#if PY_MAJOR_VERSION >= 3
+  _py_argv[0] = (wchar_t *)_program_name.c_str();
+#else
   _py_argv[0] = (char *)_program_name.c_str();
+#endif
   _py_argv[1] = NULL;
 
 #ifdef NDEBUG
@@ -78,7 +88,11 @@ P3DPythonRun(const char *program_name, const char *archive_file,
 
   // Initialize Python.  It appears to be important to do this before
   // we open the pipe streams and spawn the thread, below.
+#if PY_MAJOR_VERSION >= 3
+  Py_SetProgramName((wchar_t *)_program_name.c_str());
+#else
   Py_SetProgramName((char *)_program_name.c_str());
+#endif
   Py_Initialize();
   PyEval_InitThreads();
   PySys_SetArgv(_py_argc, _py_argv);
@@ -290,7 +304,20 @@ run_python() {
       "Send an asynchronous request to the plugin host" },
     { NULL, NULL, 0, NULL }        /* Sentinel */
   };
+
+#if PY_MAJOR_VERSION >= 3
+  static PyModuleDef p3dpython_module = {
+    PyModuleDef_HEAD_INIT,
+    "p3dpython",
+    NULL,
+    -1,
+    p3dpython_methods,
+    NULL, NULL, NULL, NULL
+  };
+  PyObject *p3dpython = PyModule_Create(&p3dpython_module);
+#else
   PyObject *p3dpython = Py_InitModule("p3dpython", p3dpython_methods);
+#endif
   if (p3dpython == NULL) {
     PyErr_Print();
     return false;
@@ -644,8 +671,11 @@ handle_pyobj_command(TiXmlElement *xcommand, bool needs_response,
           result = PyBool_FromLong(PyObject_IsTrue(obj));
 
         } else if (strcmp(method_name, "__int__") == 0) {
+#if PY_MAJOR_VERSION >= 3
+          result = PyNumber_Long(obj);
+#else
           result = PyNumber_Int(obj);
-
+#endif
         } else if (strcmp(method_name, "__float__") == 0) {
           result = PyNumber_Float(obj);
 
@@ -1510,10 +1540,12 @@ pyobj_to_xml(PyObject *value) {
     xvalue->SetAttribute("type", "bool");
     xvalue->SetAttribute("value", PyObject_IsTrue(value));
 
+#if PY_MAJOR_VERSION < 3
   } else if (PyInt_Check(value)) {
     // A plain integer value.
     xvalue->SetAttribute("type", "int");
     xvalue->SetAttribute("value", PyInt_AsLong(value));
+#endif
 
   } else if (PyLong_Check(value)) {
     // A long integer value.  This gets converted either as an integer
@@ -1535,9 +1567,21 @@ pyobj_to_xml(PyObject *value) {
     xvalue->SetAttribute("type", "float");
     xvalue->SetDoubleAttribute("value", PyFloat_AsDouble(value));
 
+
   } else if (PyUnicode_Check(value)) {
     // A unicode value.  Convert to utf-8 for the XML encoding.
     xvalue->SetAttribute("type", "string");
+
+#if PY_MAJOR_VERSION >= 3
+    // In Python 3, there are only unicode strings, and there is a
+    // handy function for getting the UTF-8 encoded version.
+    Py_ssize_t length = 0;
+    char *buffer = PyUnicode_AsUTF8AndSize(value, &length);
+    if (buffer != NULL) {
+      string str(buffer, length);
+      xvalue->SetAttribute("value", str);
+    }
+#else
     PyObject *as_str = PyUnicode_AsUTF8String(value);
     if (as_str != NULL) {
       char *buffer;
@@ -1571,6 +1615,7 @@ pyobj_to_xml(PyObject *value) {
       }
       Py_DECREF(ustr);
     }
+#endif  // PY_MAJOR_VERSION
 
   } else if (PyTuple_Check(value)) {
     // An immutable sequence.  Pass it as a concrete.
@@ -1609,7 +1654,11 @@ pyobj_to_xml(PyObject *value) {
             PyObject *as_str;
             if (PyUnicode_Check(a)) {
               // The key is a unicode value.
+#if PY_MAJOR_VERSION >= 3
+              as_str = a;
+#else
               as_str = PyUnicode_AsUTF8String(a);
+#endif
             } else {
               // The key is a string value or something else.  Make it
               // a string.
@@ -1617,7 +1666,12 @@ pyobj_to_xml(PyObject *value) {
             }
             char *buffer;
             Py_ssize_t length;
+#if PY_MAJOR_VERSION >= 3
+            buffer = PyUnicode_AsUTF8AndSize(as_str, &length);
+            if (buffer != NULL) {
+#else
             if (PyString_AsStringAndSize(as_str, &buffer, &length) != -1) {
+#endif
               string str(buffer, length);
               xitem->SetAttribute("key", str);
             }
@@ -1702,7 +1756,11 @@ xml_to_pyobj(TiXmlElement *xvalue) {
   } else if (strcmp(type, "int") == 0) {
     int value;
     if (xvalue->QueryIntAttribute("value", &value) == TIXML_SUCCESS) {
+#if PY_MAJOR_VERSION >= 3
+      return PyLong_FromLong(value);
+#else
       return PyInt_FromLong(value);
+#endif
     }
 
   } else if (strcmp(type, "float") == 0) {

direct/src/plugin/p3dPythonRun.h

@@ -159,10 +159,15 @@ private:
 
   int _session_id;
 
-  string _program_name;
   Filename _archive_file;
   int _py_argc;
-  char **_py_argv;
+#if PY_MAJOR_VERSION >= 3
+  wchar_t *_py_argv[2];
+  wstring _program_name;
+#else
+  char *_py_argv[2];
+  string _program_name;
+#endif
   bool _interactive_console;
 
   PyObject *_runner;

direct/src/showutil/FreezeTool.py

@@ -8,7 +8,7 @@ import marshal
 import imp
 import platform
 import types
-from distutils.sysconfig import PREFIX, get_python_inc, get_python_version
+from distutils.sysconfig import PREFIX, get_python_inc, get_python_version, get_config_var
 
 # Temporary (?) try..except to protect against unbuilt p3extend_frozen.
 try:
@@ -63,7 +63,7 @@ class CompilationEnvironment:
         # Paths to Python stuff.
         self.Python = None
         self.PythonIPath = get_python_inc()
-        self.PythonVersion = get_python_version()
+        self.PythonVersion = get_config_var("LDVERSION") or get_python_version()
 
         # The VC directory of Microsoft Visual Studio (if relevant)
         self.MSVC = None
@@ -970,7 +970,7 @@ class Freezer:
                 stuff = ("", "rb", imp.PY_COMPILED)
                 self.mf.load_module(mdef.moduleName, fp, pathname, stuff)
             else:
-                fp = open(pathname, modulefinder.READ_MODE)
+                fp = open(pathname, 'U')
                 stuff = ("", "r", imp.PY_SOURCE)
                 self.mf.load_module(mdef.moduleName, fp, pathname, stuff)
 
@@ -1062,7 +1062,7 @@ class Freezer:
 
     def __addPyc(self, multifile, filename, code, compressionLevel):
         if code:
-            data = imp.get_magic() + '\0\0\0\0' + \
+            data = imp.get_magic() + b'\0\0\0\0' + \
                    marshal.dumps(code)
 
             stream = StringStream(data)
@@ -1341,7 +1341,10 @@ class Freezer:
         for i in range(0, len(code), 16):
             result += '\n  '
             for c in code[i:i+16]:
-                result += ('%d,' % ord(c))
+                if isinstance(c, int): # Python 3
+                    result += ('%d,' % c)
+                else: # Python 2
+                    result += ('%d,' % ord(c))
         result += '\n};\n'
         return result
 

makepanda/makepanda.py

@@ -3271,6 +3271,7 @@ IGATEFILES=GetDirectoryContents('panda/src/downloader', ["*.h", "*_composite*.cx
 TargetAdd('libp3downloader.in', opts=OPTS, input=IGATEFILES)
 TargetAdd('libp3downloader.in', opts=['IMOD:panda3d.core', 'ILIB:libp3downloader', 'SRCDIR:panda/src/downloader'])
 TargetAdd('libp3downloader_igate.obj', input='libp3downloader.in', opts=["DEPENDENCYONLY"])
+TargetAdd('p3downloader_stringStream_ext.obj', opts=OPTS, input='stringStream_ext.cxx')
 
 #
 # DIRECTORY: panda/metalibs/pandaexpress/
@@ -3905,6 +3906,7 @@ if (not RUNTIME):
   TargetAdd('core_module.obj', opts=['IMOD:panda3d._core', 'ILIB:_core'])
 
   TargetAdd('_core.pyd', input='libp3downloader_igate.obj')
+  TargetAdd('_core.pyd', input='p3downloader_stringStream_ext.obj')
   TargetAdd('_core.pyd', input='p3express_ext_composite.obj')
   TargetAdd('_core.pyd', input='libp3express_igate.obj')
 

panda/src/downloader/stringStream.h

@@ -17,29 +17,43 @@
 
 #include "pandabase.h"
 #include "stringStreamBuf.h"
+#include "extension.h"
 
 ////////////////////////////////////////////////////////////////////
 //       Class : StringStream
 // Description : A bi-directional stream object that reads and writes
 //               data to an internal buffer, which can be retrieved
-//               and/or set as a string.
+//               and/or set as a string in Python 2 or a bytes object
+//               in Python 3.
 ////////////////////////////////////////////////////////////////////
 class EXPCL_PANDAEXPRESS StringStream : public iostream {
+public:
+  INLINE StringStream(const string &source);
+
 PUBLISHED:
+  EXTENSION(StringStream(PyObject *source));
   INLINE StringStream();
-  INLINE StringStream(const string &source);
 
   INLINE void clear_data();
   INLINE size_t get_data_size();
 
+  EXTENSION(PyObject *get_data());
+  EXTENSION(void set_data(PyObject *data));
+
+  MAKE_PROPERTY(data, get_data, set_data);
+
+public:
+#ifndef CPPPARSER
   INLINE string get_data();
   INLINE void set_data(const string &data);
+#endif
 
-public:
   INLINE void swap_data(pvector<unsigned char> &data);
 
 private:
   StringStreamBuf _buf;
+
+  friend class Extension<StringStream>;
 };
 
 #include "stringStream.I"

panda/src/downloader/stringStream_ext.cxx

@@ -0,0 +1,99 @@
+// Filename: stringStream_ext.cxx
+// Created by:  rdb (09Dec13)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#include "stringStream_ext.h"
+
+#ifdef HAVE_PYTHON
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringStream::__init__
+//       Access: Published
+//  Description:
+////////////////////////////////////////////////////////////////////
+void Extension<StringStream>::
+__init__(PyObject *source) {
+  set_data(source);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringStream::get_data
+//       Access: Published
+//  Description: Returns the contents of the data stream as a string.
+////////////////////////////////////////////////////////////////////
+PyObject *Extension<StringStream>::
+get_data() {
+  _this->flush();
+  const pvector<unsigned char> &data = _this->_buf.get_data();
+  if (!data.empty()) {
+#if PY_MAJOR_VERSION >= 3
+    return PyBytes_FromStringAndSize((char *)&data[0], data.size());
+#else
+    return PyString_FromStringAndSize((char *)&data[0], data.size());
+#endif
+  }
+#if PY_MAJOR_VERSION >= 3
+  return PyBytes_FromStringAndSize("", 0);
+#else
+  return PyString_FromStringAndSize("", 0);
+#endif
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: StringStream::set_data
+//       Access: Published
+//  Description: Replaces the contents of the data stream.  This
+//               implicitly reseeks to 0.
+////////////////////////////////////////////////////////////////////
+void Extension<StringStream>::
+set_data(PyObject *data) {
+  _this->_buf.clear();
+
+  if (data == NULL) {
+    return;
+  }
+
+#if PY_VERSION_HEX >= 0x02060000
+  if (PyObject_CheckBuffer(data)) {
+    Py_buffer view;
+    if (PyObject_GetBuffer(data, &view, PyBUF_CONTIG_RO) == -1) {
+      PyErr_SetString(PyExc_TypeError,
+                      "StringStream requires a contiguous buffer");
+      return;
+    }
+    pvector<unsigned char> pv;
+    pv.insert(pv.end(), (const unsigned char *)view.buf, (const unsigned char *)view.buf + view.len);
+    _this->_buf.swap_data(pv);
+    PyBuffer_Release(&view);
+    return;
+  }
+#endif
+
+#if PY_MAJOR_VERSION < 3
+  if (PyString_Check(data)) {
+    char *buffer;
+    Py_ssize_t length;
+    if (PyString_AsStringAndSize(data, &buffer, &length) != -1) {
+      pvector<unsigned char> pv;
+      pv.insert(pv.end(), (const unsigned char *)buffer, (const unsigned char *)buffer + length);
+      _this->_buf.swap_data(pv);
+    }
+    return;
+  }
+#endif
+
+  PyErr_SetString(PyExc_TypeError,
+                  "StringStream requires a bytes or buffer object");
+}
+
+#endif

panda/src/downloader/stringStream_ext.h

@@ -0,0 +1,43 @@
+// Filename: stringStream_ext.h
+// Created by:  rdb (06Aug15)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef STRINGSTREAM_EXT_H
+#define STRINGSTREAM_EXT_H
+
+#include "dtoolbase.h"
+
+#ifdef HAVE_PYTHON
+
+#include "extension.h"
+#include "stringStream.h"
+#include "py_panda.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : Extension<StringStream>
+// Description : This class defines the extension methods for
+//               StringStream, which are called instead of
+//               any C++ methods with the same prototype.
+////////////////////////////////////////////////////////////////////
+template<>
+class Extension<StringStream> : public ExtensionBase<StringStream> {
+public:
+  void __init__(PyObject *source);
+
+  PyObject *get_data();
+  void set_data(PyObject *data);
+};
+
+#endif  // HAVE_PYTHON
+
+#endif  // STRINGSTREAM_EXT_H