deploy-ng: new extensible blob format with PRC configurability

direct/src/showutil/FreezeTool.py

@@ -8,10 +8,12 @@ import marshal
 import imp
 import platform
 import struct
-from io import StringIO, TextIOWrapper
+from io import StringIO, BytesIO, TextIOWrapper
 import distutils.sysconfig as sysconf
 import zipfile
 
+from . import pefile
+
 # Temporary (?) try..except to protect against unbuilt p3extend_frozen.
 try:
     import p3extend_frozen
@@ -1589,7 +1591,7 @@ class Freezer:
 
         return target
 
-    def generateRuntimeFromStub(self, basename, stub_file):
+    def generateRuntimeFromStub(self, basename, stub_file, fields={}):
         # We must have a __main__ module to make an exe file.
         if not self.__writingModule('__main__'):
             message = "Can't generate an executable without a __main__ module."
@@ -1602,15 +1604,18 @@ class Freezer:
             target = basename
             modext = '.so'
 
-        address_offset = 0
-
-        # First gather up the strings for all the module names.
-        blob = b""
+        # First gather up the strings and code for all the module names, and
+        # put those in a string pool.
+        pool = b""
         strings = set()
 
         for moduleName, mdef in self.getModuleDefs():
             strings.add(moduleName.encode('ascii'))
 
+        for value in fields.values():
+            if value is not None:
+                strings.add(value.encode('utf-8'))
+
         # Sort by length descending, allowing reuse of partial strings.
         strings = sorted(strings, key=lambda str:-len(str))
         string_offsets = {}
@@ -1618,13 +1623,15 @@ class Freezer:
         for string in strings:
             # First check whether it's already in there; it could be part of
             # a longer string.
-            offset = blob.find(string + b'\0')
+            offset = pool.find(string + b'\0')
             if offset < 0:
-                offset = len(blob)
-                blob += string + b'\0'
+                offset = len(pool)
+                pool += string + b'\0'
             string_offsets[string] = offset
 
-        # Generate export table.
+        # Now go through the modules and add them to the pool as well.  These
+        # are not 0-terminated, but we later record their sizes and names in
+        # a table after the blob header.
         moduleList = []
 
         for moduleName, mdef in self.getModuleDefs():
@@ -1635,9 +1642,9 @@ class Freezer:
                 continue
 
             # For whatever it's worth, align the code blocks.
-            if len(blob) & 3 != 0:
-                pad = (4 - (len(blob) & 3))
-                blob += b'\0' * pad
+            if len(pool) & 3 != 0:
+                pad = (4 - (len(pool) & 3))
+                pool += b'\0' * pad
 
             assert not mdef.exclude
             # Allow importing this module.
@@ -1649,8 +1656,8 @@ class Freezer:
                 if getattr(module, "__path__", None):
                     # Indicate package by negative size
                     size = -size
-                moduleList.append((moduleName, len(blob), size))
-                blob += code
+                moduleList.append((moduleName, len(pool), size))
+                pool += code
                 continue
 
             # This is a module with no associated Python code.  It is either
@@ -1668,55 +1675,327 @@ class Freezer:
                 code = 'import sys;del sys.modules["%s"];import sys,os,imp;imp.load_dynamic("%s",os.path.join(os.path.dirname(sys.executable), "%s%s"))' % (moduleName, moduleName, moduleName, modext)
                 code = compile(code, moduleName, 'exec')
                 code = marshal.dumps(code)
-                moduleList.append((moduleName, len(blob), len(code)))
-                blob += code
-
-        # Now compose the final blob.
-        if '64' in self.platform:
-            layout = '<QQixxxx'
+                moduleList.append((moduleName, len(pool), len(code)))
+                pool += code
+
+        # Determine the format of the header and module list entries depending
+        # on the platform.
+        num_pointers = 10
+        stub_data = bytearray(stub_file.read())
+        if self._is_executable_64bit(stub_data):
+            header_layout = '<QQHHHH8x%dQQ' % num_pointers
+            entry_layout = '<QQixxxx'
         else:
-            layout = '<IIi'
+            header_layout = '<QQHHHH8x%dII' % num_pointers
+            entry_layout = '<IIi'
 
-        blob2 = bytes()
-        blobOffset = (len(moduleList) + 1) * struct.calcsize(layout)
-        blobOffset += address_offset
+        # Calculate the size of the header and module table, so that we can
+        # determine the proper offset for the string pointers.
+        pool_offset = (len(moduleList) + 1) * struct.calcsize(entry_layout)
 
+        # The module table is the first thing in the blob.
+        blob = b""
         for moduleName, offset, size in moduleList:
             encoded = moduleName.encode('ascii')
-            stringOffset = blobOffset + string_offsets[encoded]
+            string_offset = pool_offset + string_offsets[encoded]
             if size != 0:
-                offset += blobOffset
-            blob2 += struct.pack(layout, stringOffset, offset, size)
+                offset += pool_offset
+            blob += struct.pack(entry_layout, string_offset, offset, size)
 
-        blob2 += struct.pack(layout, 0, 0, 0)
-        assert len(blob2) + address_offset == blobOffset
+        blob += struct.pack(entry_layout, 0, 0, 0)
 
-        blob = blob2 + blob
-        del blob2
+        # Add the string pool.
+        assert len(blob) == pool_offset
+        blob += pool
+        del pool
 
         # Total blob length should be aligned to 8 bytes.
         if len(blob) & 7 != 0:
             pad = (8 - (len(blob) & 7))
             blob += b'\0' * pad
 
-        # Build from pre-built binary stub
-        with open(target, 'wb') as f:
-            f.write(stub_file.read())
-            offset = f.tell()
+        if self.platform.startswith('win'):
+            # We don't use mmap on Windows.
+            blob_align = 32
+        else:
+            # Align to page size, so that it can be mmapped.
+            blob_align = 4096
 
+        # Determine the blob offset, padding the stub if necessary.
+        blob_offset = len(stub_data)
+        if (blob_offset & (blob_align - 1)) != 0:
             # Add padding to align to the page size, so it can be mmapped.
-            if offset % 4095 != 0:
-                pad = (4096 - (offset & 4095))
-                f.write(b'\0' * pad)
-                offset += pad
+            pad = (blob_align - (blob_offset & (blob_align - 1)))
+            stub_data += (b'\0' * pad)
+            blob_offset += pad
+        assert (blob_offset % blob_align) == 0
+        assert blob_offset == len(stub_data)
+
+        # Calculate the offsets for the variables.  These are pointers,
+        # relative to the beginning of the blob.
+        field_offsets = {}
+        for key, value in fields.items():
+            if value is not None:
+                encoded = value.encode('utf-8')
+                field_offsets[key] = pool_offset + string_offsets[encoded]
+
+        # Compose the header we will be writing to the stub, to tell it where
+        # to find the module data blob, as well as other variables.
+        header = struct.pack(header_layout,
+            blob_offset,
+            len(blob),
+            1, # Version number
+            num_pointers, # Number of pointers that follow
+            0, # Codepage, not yet used
+            0, # Flags, not yet used
+            0, # Module table pointer, always 0 for now.
+            # The following variables need to be set before static init
+            # time.  See configPageManager.cxx, where they are read.
+            field_offsets.get('prc_data', 0),
+            field_offsets.get('default_prc_dir', 0),
+            field_offsets.get('prc_dir_envvars', 0),
+            field_offsets.get('prc_path_envvars', 0),
+            field_offsets.get('prc_patterns', 0),
+            field_offsets.get('prc_encrypted_patterns', 0),
+            field_offsets.get('prc_encryption_key', 0),
+            field_offsets.get('prc_executable_patterns', 0),
+            field_offsets.get('prc_executable_args_envvar', 0),
+            0)
+
+        # Now, find the location of the 'blobinfo' symbol in the executable,
+        # to which we will write our header.
+        if not self._replace_symbol(stub_data, b'blobinfo', header):
+            # This must be a legacy deploy-stub, which requires the offset to
+            # be appended to the end.
+            blob += struct.pack('<Q', blob_offset)
 
+        with open(target, 'wb') as f:
+            f.write(stub_data)
+            assert f.tell() == blob_offset
             f.write(blob)
-            # And tack on the offset of the blob to the end.
-            f.write(struct.pack('<Q', offset))
 
         os.chmod(target, 0o755)
         return target
 
+    def _is_executable_64bit(self, data):
+        """Returns true if this is a 64-bit executable."""
+
+        if data.startswith(b'MZ'):
+            # A Windows PE file.
+            offset, = struct.unpack_from('<I', data, 0x3c)
+            assert data[offset:offset+4] == b'PE\0\0'
+
+            magic, = struct.unpack_from('<H', data, offset + 24)
+            assert magic in (0x010b, 0x020b)
+            return magic == 0x020b
+
+        elif data.startswith(b"\177ELF"):
+            # A Linux/FreeBSD ELF executable.
+            elfclass = ord(data[4:5])
+            assert elfclass in (1, 2)
+            return elfclass == 2
+
+        elif data[:4] in (b'\xFE\xED\xFA\xCE', b'\xCE\xFA\xED\xFE'):
+            # 32-bit Mach-O file, as used on macOS.
+            return False
+
+        elif data[:4] in (b'\xFE\xED\xFA\xCF', b'\xCF\xFA\xED\xFE'):
+            # 64-bit Mach-O file, as used on macOS.
+            return True
+
+        elif data[:4] in (b'\xCA\xFE\xBA\xBE', b'\xBE\xBA\xFE\xCA',
+                          b'\xCA\xFE\xBA\xBF', b'\xBF\xBA\xFE\xCA'):
+            # A universal file, containing one or more Mach-O files.
+            #TODO: how to handle universal stubs with more than one arch?
+            pass
+
+    def _replace_symbol(self, data, symbol_name, replacement):
+        """We store a custom section in the binary file containing a header
+        containing offsets to the binary data."""
+
+        if data.startswith(b'MZ'):
+            # A Windows PE file.
+            pe = pefile.PEFile()
+            pe.read(BytesIO(data))
+            addr = pe.get_export_address(symbol_name)
+            if addr is not None:
+                # We found it, return its offset in the file.
+                offset = pe.get_address_offset(addr)
+                if offset is not None:
+                    data[offset:offset+len(replacement)] = replacement
+                    return True
+
+        elif data.startswith(b"\177ELF"):
+            return self._replace_symbol_elf(data, symbol_name, replacement)
+
+        elif data[:4] in (b'\xFE\xED\xFA\xCE', b'\xCE\xFA\xED\xFE',
+                          b'\xFE\xED\xFA\xCF', b'\xCF\xFA\xED\xFE'):
+            off = self._find_symbol_macho(macho_data, symbol_name)
+            if off is not None:
+                data[off:off+len(replacement)] = replacement
+                return True
+            return False
+
+        elif data[:4] in (b'\xCA\xFE\xBA\xBE', b'\xBE\xBA\xFE\xCA'):
+            # Universal binary with 32-bit offsets.
+            return self._replace_symbol_fat(data, b'_' + symbol_name, replacement, False)
+
+        elif data[:4] in (b'\xCA\xFE\xBA\xBF', b'\xBF\xBA\xFE\xCA'):
+            # Universal binary with 64-bit offsets.
+            return self._replace_symbol_fat(data, b'_' + symbol_name, replacement, True)
+
+        # We don't know what kind of file this is.
+        return False
+
+    def _replace_symbol_elf(self, elf_data, symbol_name, replacement):
+        """ The Linux/FreeBSD implementation of _replace_symbol. """
+
+        replaced = False
+
+        # Make sure we read in the correct endianness and integer size
+        endian = "<>"[ord(elf_data[5:6]) - 1]
+        is_64bit = ord(elf_data[4:5]) - 1 # 0 = 32-bits, 1 = 64-bits
+        header_struct = endian + ("HHIIIIIHHHHHH", "HHIQQQIHHHHHH")[is_64bit]
+        section_struct = endian + ("4xI4xIIII8xI", "4xI8xQQQI12xQ")[is_64bit]
+        symbol_struct = endian + ("IIIBBH", "IBBHQQ")[is_64bit]
+
+        header_size = struct.calcsize(header_struct)
+        type, machine, version, entry, phoff, shoff, flags, ehsize, phentsize, phnum, shentsize, shnum, shstrndx \
+          = struct.unpack_from(header_struct, elf_data, 16)
+        section_offsets = []
+        symbol_tables = []
+        string_tables = {}
+
+        # Seek to the section header table and find the symbol tables.
+        ptr = shoff
+        for i in range(shnum):
+            type, addr, offset, size, link, entsize = struct.unpack_from(section_struct, elf_data[ptr:ptr+shentsize])
+            ptr += shentsize
+            section_offsets.append(offset - addr)
+            if type == 0x0B and link != 0: # SHT_DYNSYM, links to string table
+                symbol_tables.append((offset, size, link, entsize))
+                string_tables[link] = None
+
+        # Read the relevant string tables.
+        for idx in list(string_tables.keys()):
+            ptr = shoff + idx * shentsize
+            type, addr, offset, size, link, entsize = struct.unpack_from(section_struct, elf_data[ptr:ptr+shentsize])
+            if type == 3:
+                string_tables[idx] = elf_data[offset:offset+size]
+
+        # Loop through to find the offset of the "blobinfo" symbol.
+        for offset, size, link, entsize in symbol_tables:
+            entries = size // entsize
+            for i in range(entries):
+                ptr = offset + i * entsize
+                fields = struct.unpack_from(symbol_struct, elf_data[ptr:ptr+entsize])
+                if is_64bit:
+                    name, info, other, shndx, value, size = fields
+                else:
+                    name, value, size, info, other, shndx = fields
+
+                if not name:
+                    continue
+
+                name = string_tables[link][name : string_tables[link].find(b'\0', name)]
+                if name == symbol_name:
+                    if shndx == 0: # SHN_UNDEF
+                        continue
+                    elif shndx >= 0xff00 and shndx <= 0xffff:
+                        assert False
+                    else:
+                        # Got it.  Make the replacement.
+                        off = section_offsets[shndx] + value
+                        elf_data[off:off+len(replacement)] = replacement
+                        replaced = True
+
+        return replaced
+
+    def _find_symbol_macho(self, macho_data, symbol_name):
+        """ Returns the offset of the given symbol in the binary file. """
+
+        if macho_data[:4] in (b'\xCE\xFA\xED\xFE', b'\xCF\xFA\xED\xFE'):
+            endian = '<'
+        else:
+            endian = '>'
+
+        cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = \
+            struct.unpack_from(endian + 'IIIIII', macho_data, 4)
+
+        is_64bit = (cputype & 0x1000000) != 0
+        segments = []
+
+        cmd_ptr = 28
+        nlist_struct = endian + 'IBBHI'
+        if is_64bit:
+            nlist_struct = endian + 'IBBHQ'
+            cmd_ptr += 4
+        nlist_size = struct.calcsize(nlist_struct)
+
+        for i in range(ncmds):
+            cmd, cmd_size = struct.unpack_from(endian + 'II', macho_data, cmd_ptr)
+            cmd_data = macho_data[cmd_ptr+8:cmd_ptr+cmd_size]
+            cmd_ptr += cmd_size
+
+            cmd &= ~0x80000000
+
+            if cmd == 0x01: # LC_SEGMENT
+                segname, vmaddr, vmsize, fileoff, filesize, maxprot, initprot, nsects, flags = \
+                    struct.unpack_from(endian + '16sIIIIIIII', cmd_data)
+                segments.append((vmaddr, vmsize, fileoff))
+
+            elif cmd == 0x19: # LC_SEGMENT_64
+                segname, vmaddr, vmsize, fileoff, filesize, maxprot, initprot, nsects, flags = \
+                    struct.unpack_from(endian + '16sQQQQIIII', cmd_data)
+                segments.append((vmaddr, vmsize, fileoff))
+
+            elif cmd == 0x2: # LC_SYMTAB
+                symoff, nsyms, stroff, strsize = \
+                    struct.unpack_from(endian + 'IIII', cmd_data)
+
+                strings = macho_data[stroff:stroff+strsize]
+
+                for i in range(nsyms):
+                    strx, type, sect, desc, value = struct.unpack_from(nlist_struct, macho_data, symoff)
+                    symoff += nlist_size
+                    name = strings[strx : strings.find(b'\0', strx)]
+
+                    if name == symbol_name:
+                        # Find out in which segment this is.
+                        for vmaddr, vmsize, fileoff in segments:
+                            # Is it defined in this segment?
+                            rel = value - vmaddr
+                            if rel >= 0 and rel < vmsize:
+                                # Yes, so return the symbol offset.
+                                return fileoff + rel
+
+    def _replace_symbol_fat(self, fat_data, symbol_name, replacement, is_64bit):
+        """ Implementation of _replace_symbol for universal binaries. """
+        num_fat, = struct.unpack_from('>I', fat_data, 4)
+
+        # After the header we get a table of executables in this fat file,
+        # each one with a corresponding offset into the file.
+        replaced = False
+        ptr = 8
+        for i in range(num_fat):
+            if is_64bit:
+                cputype, cpusubtype, offset, size, align = \
+                    struct.unpack_from('>QQQQQ', fat_data, ptr)
+                ptr += 40
+            else:
+                cputype, cpusubtype, offset, size, align = \
+                    struct.unpack_from('>IIIII', fat_data, ptr)
+                ptr += 20
+
+            macho_data = fat_data[offset:offset+size]
+            off = self._find_symbol_macho(macho_data, symbol_name)
+            if off is not None:
+                off += offset
+                fat_data[off:off+len(replacement)] = replacement
+                replaced = True
+
+        return replaced
+
     def makeModuleDef(self, mangledName, code):
         result = ''
         result += 'static unsigned char %s[] = {' % (mangledName)

direct/src/showutil/dist.py

@@ -225,7 +225,17 @@ class build_apps(distutils.core.Command):
                 stub_path = os.path.join(os.path.dirname(dtool_path), '..', 'bin', stub_name)
                 stub_file = open(stub_path, 'rb')
 
-            freezer.generateRuntimeFromStub(os.path.join(builddir, appname), stub_file)
+            freezer.generateRuntimeFromStub(os.path.join(builddir, appname), stub_file, {
+                'prc_data': None,
+                'default_prc_dir': None,
+                'prc_dir_envvars': None,
+                'prc_path_envvars': None,
+                'prc_patterns': None,
+                'prc_encrypted_patterns': None,
+                'prc_encryption_key': None,
+                'prc_executable_patterns': None,
+                'prc_executable_args_envvar': None,
+            })
             stub_file.close()
 
             freezer_extras.update(freezer.extras)
@@ -492,16 +502,22 @@ class build_apps(distutils.core.Command):
             # Elf magic.  Used on (among others) Linux and FreeBSD.
             deps = self._read_dependencies_elf(fp, os.path.dirname(source_path), search_path)
 
-        elif magic in (b'\xFE\xED\xFA\xCE', b'\xCE\xFA\xED\xFE',
-                       b'\xFE\xED\xFA\xCF', b'\xCF\xFA\xED\xFE'):
+        elif magic in (b'\xCE\xFA\xED\xFE', b'\xCF\xFA\xED\xFE'):
             # A Mach-O file, as used on macOS.
-            deps = self._read_dependencies_macho(fp)
+            deps = self._read_dependencies_macho(fp, '<')
 
-        elif magic in (b'\xCA\xFE\xBA\xBE', b'\xBE\xBA\xFE\bCA'):
+        elif magic in (b'\xFE\xED\xFA\xCE', b'\xFE\xED\xFA\xCF'):
+            deps = self._read_dependencies_macho(fp, '>')
+
+        elif magic in (b'\xCA\xFE\xBA\xBE', b'\xBE\xBA\xFE\xCA'):
             # A fat file, containing multiple Mach-O binaries.  In the future,
             # we may want to extract the one containing the architecture we
             # are building for.
-            deps = self._read_dependencies_fat(fp)
+            deps = self._read_dependencies_fat(fp, False)
+
+        elif magic in (b'\xCA\xFE\xBA\xBF', b'\xBF\xBA\xFE\xCA'):
+            # A 64-bit fat file.
+            deps = self._read_dependencies_fat(fp, True)
 
         # If we discovered any dependencies, recursively add those.
         if deps:
@@ -573,23 +589,23 @@ class build_apps(distutils.core.Command):
         search_path += rpath
         return needed
 
-    def _read_dependencies_macho(self, fp):
+    def _read_dependencies_macho(self, fp, endian):
         """ Having read the first 4 bytes of the Mach-O file, fetches the
         dependent libraries and returns those as a list. """
 
         cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = \
-            struct.unpack('<IIIIII', fp.read(24))
+            struct.unpack(endian + 'IIIIII', fp.read(24))
 
-        is_64 = (cputype & 0x1000000) != 0
-        if is_64:
+        is_64bit = (cputype & 0x1000000) != 0
+        if is_64bit:
             fp.read(4)
 
         # After the header, we get a series of linker commands.  We just
         # iterate through them and gather up the LC_LOAD_DYLIB commands.
         load_dylibs = []
         for i in range(ncmds):
-            cmd, cmdsize = struct.unpack('<II', fp.read(8))
-            cmd_data = fp.read(cmdsize - 8)
+            cmd, cmd_size = struct.unpack(endian + 'II', fp.read(8))
+            cmd_data = fp.read(cmd_size - 8)
             cmd &= ~0x80000000
 
             if cmd == 0x0c: # LC_LOAD_DYLIB
@@ -600,20 +616,41 @@ class build_apps(distutils.core.Command):
 
         return load_dylibs
 
-    def _read_dependencies_fat(self, fp):
-        num_fat = struct.unpack('>I', fp.read(4))[0]
-        if num_fat == 0:
-            return []
+    def _read_dependencies_fat(self, fp, is_64bit):
+        num_fat, = struct.unpack('>I', fp.read(4))
 
         # After the header we get a table of executables in this fat file,
         # each one with a corresponding offset into the file.
-        # We are just interested in the first one for now.
-        cputype, cpusubtype, offset, size, align = \
-            struct.unpack('>IIIII', fp.read(20))
+        offsets = []
+        for i in range(num_fat):
+            if is_64bit:
+                cputype, cpusubtype, offset, size, align = \
+                    struct.unpack('>QQQQQ', fp.read(40))
+            else:
+                cputype, cpusubtype, offset, size, align = \
+                    struct.unpack('>IIIII', fp.read(20))
+            offsets.append(offset)
+
+        # Go through each of the binaries in the fat file.
+        deps = []
+        for offset in offsets:
+            # Add 4, since it expects we've already read the magic.
+            fp.seek(offset)
+            magic = fp.read(4)
+
+            if magic in (b'\xCE\xFA\xED\xFE', b'\xCF\xFA\xED\xFE'):
+                endian = '<'
+            elif magic in (b'\xFE\xED\xFA\xCE', b'\xFE\xED\xFA\xCF'):
+                endian = '>'
+            else:
+                # Not a Mach-O file we can read.
+                continue
+
+            for dep in self._read_dependencies_macho(fp, endian):
+                if dep not in deps:
+                    deps.append(dep)
 
-        # Add 4, since it expects we've already read the magic.
-        fp.seek(offset + 4)
-        return self._read_dependencies_macho(fp)
+        return deps
 
 
 class bdist_apps(distutils.core.Command):

direct/src/showutil/pefile.py

@@ -19,6 +19,7 @@ if sys.version_info >= (3, 0):
 # Define some internally used structures.
 RVASize = namedtuple('RVASize', ('addr', 'size'))
 impdirtab = namedtuple('impdirtab', ('lookup', 'timdat', 'forward', 'name', 'impaddr'))
+expdirtab = namedtuple('expdirtab', ('flags', 'timdat', 'majver', 'minver', 'name', 'ordinal_base', 'nentries', 'nnames', 'entries', 'names', 'ordinals'))
 
 
 def _unpack_zstring(mem, offs=0):
@@ -571,10 +572,11 @@ class PEFile(object):
 
         # Read the sections into some kind of virtual memory.
         self.vmem = bytearray(self.sections[-1].vaddr + self.sections[-1].size)
+        memview = memoryview(self.vmem)
 
         for section in self.sections:
             fp.seek(section.offset)
-            fp.readinto(memoryview(self.vmem)[section.vaddr:section.vaddr+section.size])
+            fp.readinto(memview[section.vaddr:section.vaddr+section.size])
 
         # Read the import table.
         start = self.imp_rva.addr
@@ -596,6 +598,43 @@ class PEFile(object):
         if self.res_rva.addr and self.res_rva.size:
             self.resources.unpack_from(self.vmem, self.res_rva.addr)
 
+    def get_export_address(self, symbol_name):
+        """ Finds the virtual address for a named export symbol. """
+
+        start = self.exp_rva.addr
+        expdir = expdirtab(*unpack('<IIHHIIIIIII', self.vmem[start:start+40]))
+        if expdir.nnames == 0 or expdir.ordinals == 0 or expdir.names == 0:
+            return None
+
+        nptr = expdir.names
+        optr = expdir.ordinals
+        for i in range(expdir.nnames):
+            name_rva, = unpack('<I', self.vmem[nptr:nptr+4])
+            ordinal, = unpack('<H', self.vmem[optr:optr+2])
+            if name_rva != 0:
+                name = _unpack_zstring(self.vmem, name_rva)
+                if name == symbol_name:
+                    assert ordinal >= 0 and ordinal < expdir.nentries
+                    start = expdir.entries + 8 * ordinal
+                    addr, = unpack('<I', self.vmem[start:start+4])
+                    return addr
+            nptr += 4
+            optr += 2
+
+    def get_address_offset(self, addr):
+        """ Turns an address into a offset relative to the file beginning. """
+
+        section = self.get_address_section(addr)
+        if section is not None:
+            return (addr - section.vaddr) + section.offset
+
+    def get_address_section(self, addr):
+        """ Returns the section that this virtual address belongs to. """
+
+        for section in self.sections:
+            if addr >= section.vaddr and addr < section.vaddr + section.size:
+                return section
+
     def add_icon(self, icon, ordinal=2):
         """ Adds an icon resource from the given Icon object.  Requires
         calling add_resource_section() afterwards. """

dtool/src/prc/configPageManager.cxx

@@ -37,6 +37,10 @@
 #include <algorithm>
 #include <ctype.h>
 
+#ifndef _MSC_VER
+#include <dlfcn.h>
+#endif
+
 ConfigPageManager *ConfigPageManager::_global_ptr = NULL;
 
 /**
@@ -90,11 +94,44 @@ reload_implicit_pages() {
   }
   _implicit_pages.clear();
 
+  // If we are running inside a deployed application, see if it exposes
+  // information about how the PRC data should be initialized.
+  struct BlobInfo {
+    uint64_t blob_offset;
+    uint64_t blob_size;
+    uint16_t version;
+    uint16_t num_pointers;
+    uint16_t codepage;
+    uint16_t flags;
+    uint64_t reserved;
+    const void *module_table;
+    const char *prc_data;
+    const char *default_prc_dir;
+    const char *prc_dir_envvars;
+    const char *prc_path_envvars;
+    const char *prc_patterns;
+    const char *prc_encrypted_patterns;
+    const char *prc_encryption_key;
+    const char *prc_executable_patterns;
+    const char *prc_executable_args_envvar;
+  };
+#ifdef _MSC_VER
+  const BlobInfo *blobinfo = (const BlobInfo *)GetProcAddress(GetModuleHandle(NULL), "blobinfo");
+#else
+  const BlobInfo *blobinfo = (const BlobInfo *)dlsym(RTLD_SELF, "blobinfo");
+#endif
+  if (blobinfo != nullptr && (blobinfo->version == 0 || blobinfo->num_pointers < 10)) {
+    blobinfo = nullptr;
+  }
+
   // PRC_PATTERNS lists one or more filename templates separated by spaces.
   // Pull them out and store them in _prc_patterns.
   _prc_patterns.clear();
 
   string prc_patterns = PRC_PATTERNS;
+  if (blobinfo != nullptr && blobinfo->prc_patterns != nullptr) {
+    prc_patterns = blobinfo->prc_patterns;
+  }
   if (!prc_patterns.empty()) {
     vector_string pat_list;
     ConfigDeclaration::extract_words(prc_patterns, pat_list);
@@ -114,6 +151,9 @@ reload_implicit_pages() {
   _prc_encrypted_patterns.clear();
 
   string prc_encrypted_patterns = PRC_ENCRYPTED_PATTERNS;
+  if (blobinfo != nullptr && blobinfo->prc_encrypted_patterns != nullptr) {
+    prc_encrypted_patterns = blobinfo->prc_encrypted_patterns;
+  }
   if (!prc_encrypted_patterns.empty()) {
     vector_string pat_list;
     ConfigDeclaration::extract_words(prc_encrypted_patterns, pat_list);
@@ -131,6 +171,9 @@ reload_implicit_pages() {
   _prc_executable_patterns.clear();
 
   string prc_executable_patterns = PRC_EXECUTABLE_PATTERNS;
+  if (blobinfo != nullptr && blobinfo->prc_executable_patterns != nullptr) {
+    prc_executable_patterns = blobinfo->prc_executable_patterns;
+  }
   if (!prc_executable_patterns.empty()) {
     vector_string pat_list;
     ConfigDeclaration::extract_words(prc_executable_patterns, pat_list);
@@ -151,6 +194,9 @@ reload_implicit_pages() {
   // spaces.  Pull them out, and each of those contains the name of a single
   // directory to search.  Add it to the search path.
   string prc_dir_envvars = PRC_DIR_ENVVARS;
+  if (blobinfo != nullptr && blobinfo->prc_dir_envvars != nullptr) {
+    prc_dir_envvars = blobinfo->prc_dir_envvars;
+  }
   if (!prc_dir_envvars.empty()) {
     vector_string prc_dir_envvar_list;
     ConfigDeclaration::extract_words(prc_dir_envvars, prc_dir_envvar_list);
@@ -170,6 +216,9 @@ reload_implicit_pages() {
   // spaces.  Pull them out, and then each one of those contains a list of
   // directories to search.  Add each of those to the search path.
   string prc_path_envvars = PRC_PATH_ENVVARS;
+  if (blobinfo != nullptr && blobinfo->prc_path_envvars != nullptr) {
+    prc_path_envvars = blobinfo->prc_path_envvars;
+  }
   if (!prc_path_envvars.empty()) {
     vector_string prc_path_envvar_list;
     ConfigDeclaration::extract_words(prc_path_envvars, prc_path_envvar_list);
@@ -201,7 +250,7 @@ reload_implicit_pages() {
  * DEFAULT_PATHSEP.
  */
   string prc_path2_envvars = PRC_PATH2_ENVVARS;
-  if (!prc_path2_envvars.empty()) {
+  if (!prc_path2_envvars.empty() && blobinfo == nullptr) {
     vector_string prc_path_envvar_list;
     ConfigDeclaration::extract_words(prc_path2_envvars, prc_path_envvar_list);
     for (size_t i = 0; i < prc_path_envvar_list.size(); ++i) {
@@ -225,6 +274,9 @@ reload_implicit_pages() {
     // If nothing's on the search path (PRC_DIR and PRC_PATH were not
     // defined), then use the DEFAULT_PRC_DIR.
     string default_prc_dir = DEFAULT_PRC_DIR;
+    if (blobinfo != nullptr && blobinfo->default_prc_dir != nullptr) {
+      default_prc_dir = blobinfo->default_prc_dir;
+    }
     if (!default_prc_dir.empty()) {
       // It's already from-os-specific by ppremake.
       Filename prc_dir_filename = default_prc_dir;
@@ -297,12 +349,24 @@ reload_implicit_pages() {
     }
   }
 
+  int i = 1;
+
+  // If prc_data is predefined, we load it as an implicit page.
+  if (blobinfo != nullptr && blobinfo->prc_data != nullptr) {
+    ConfigPage *page = new ConfigPage("builtin", true, i);
+    ++i;
+    _implicit_pages.push_back(page);
+    _pages_sorted = false;
+
+    istringstream in(blobinfo->prc_data);
+    page->read_prc(in);
+  }
+
   // Now we have a list of filenames in order from most important to least
   // important.  Walk through the list in reverse order to load their
   // contents, because we want the first file in the list (the most important)
   // to be on the top of the stack.
   ConfigFiles::reverse_iterator ci;
-  int i = 1;
   for (ci = config_files.rbegin(); ci != config_files.rend(); ++ci) {
     const ConfigFile &file = (*ci);
     Filename filename = file._filename;
@@ -313,6 +377,9 @@ reload_implicit_pages() {
       string command = filename.to_os_specific();
 
       string envvar = PRC_EXECUTABLE_ARGS_ENVVAR;
+      if (blobinfo != nullptr && blobinfo->prc_executable_args_envvar != nullptr) {
+        envvar = blobinfo->prc_executable_args_envvar;
+      }
       if (!envvar.empty()) {
         string args = ExecutionEnvironment::get_environment_variable(envvar);
         if (!args.empty()) {
@@ -343,7 +410,11 @@ reload_implicit_pages() {
         _implicit_pages.push_back(page);
         _pages_sorted = false;
 
-        page->read_encrypted_prc(in, PRC_ENCRYPTION_KEY);
+        if (blobinfo != nullptr && blobinfo->prc_encryption_key != nullptr) {
+          page->read_encrypted_prc(in, blobinfo->prc_encryption_key);
+        } else {
+          page->read_encrypted_prc(in, PRC_ENCRYPTION_KEY);
+        }
       }
 
     } else if ((file._file_flags & FF_read) != 0) {

makepanda/makepanda.py

@@ -6583,7 +6583,7 @@ if PkgSkip("PYTHON") == 0:
     if GetTarget() == 'windows':
         TargetAdd('frozen_dllmain.obj', opts=OPTS, input='frozen_dllmain.c')
 
-    if GetTarget() == 'linux':
+    if GetTarget() == 'linux' or GetTarget() == 'freebsd':
         # Setup rpath so libs can be found in the same directory as the deployed game
         LibName('DEPLOYSTUB', "-Wl,-rpath,\$ORIGIN")
         LibName('DEPLOYSTUB', "-Wl,-z,origin")

pandatool/src/deploy-stub/deploy-stub.c

@@ -22,6 +22,29 @@
 #endif
 #endif
 
+/* Define an exposed symbol where we store the offset to the module data. */
+#ifdef _MSC_VER
+__declspec(dllexport)
+#else
+__attribute__((__visibility__("default"), used))
+#endif
+volatile struct {
+  uint64_t blob_offset;
+  uint64_t blob_size;
+  uint16_t version;
+  uint16_t num_pointers;
+  uint16_t codepage;
+  uint16_t flags;
+  uint64_t reserved;
+
+  // Leave room for future expansion.  We only read pointer 0, but there are
+  // other pointers that are being read by configPageManager.cxx.
+  void *pointers[24];
+
+  // The reason we initialize it to -1 is because otherwise, smart linkers may
+  // end up putting it in the .bss section for zero-initialized data.
+} blobinfo = {(uint64_t)-1};
+
 #ifdef MS_WINDOWS
 #define WIN32_LEAN_AND_MEAN
 #include <windows.h>
@@ -221,15 +244,12 @@ error:
     return sts;
 }
 
-
-#if defined(_WIN32) && PY_MAJOR_VERSION >= 3
-int wmain(int argc, wchar_t *argv[]) {
-#else
-int main(int argc, char *argv[]) {
-#endif
-  struct _frozen *blob, *moddef;
-  uint64_t begin, end, size;
-  int retval;
+/**
+ * Maps the binary blob at the given memory address to memory, and returns the
+ * pointer to the beginning of it.
+ */
+static void *map_blob(off_t offset, size_t size) {
+  void *blob;
   FILE *runtime;
 
 #ifdef _WIN32
@@ -243,7 +263,7 @@ int main(int argc, char *argv[]) {
   mib[3] = getpid();
   if (sysctl(mib, 4, (void *)buffer, &bufsize, NULL, 0) == -1) {
     perror("sysctl");
-    return 1;
+    return NULL;
   }
   runtime = fopen(buffer, "rb");
 #else
@@ -251,46 +271,108 @@ int main(int argc, char *argv[]) {
   runtime = fopen(argv[0], "rb");
 #endif
 
-  // Get offsets
-  fseek(runtime, -8, SEEK_END);
-  end = ftell(runtime);
-  fread(&begin, 8, 1, runtime);
-  size = end - begin;
+  // Get offsets.  In version 0, we read it from the end of the file.
+  if (blobinfo.version == 0) {
+    uint64_t end, begin;
+    fseek(runtime, -8, SEEK_END);
+    end = ftell(runtime);
+    fread(&begin, 8, 1, runtime);
+
+    offset = (off_t)begin;
+    size = (size_t)(end - begin);
+  }
 
   // mmap the section indicated by the offset (or malloc/fread on windows)
 #ifdef _WIN32
-  blob = (struct _frozen *)malloc(size);
+  blob = (void *)malloc(size);
   assert(blob != NULL);
-  fseek(runtime, (long)begin, SEEK_SET);
+  fseek(runtime, (long)offset, SEEK_SET);
   fread(blob, size, 1, runtime);
 #else
-  blob = (struct _frozen *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fileno(runtime), begin);
-  assert(blob != NULL);
+  blob = (void *)mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fileno(runtime), offset);
+  assert(blob != MAP_FAILED);
 #endif
 
   fclose(runtime);
+  return blob;
+}
+
+/**
+ * The inverse of map_blob.
+ */
+static void unmap_blob(void *blob) {
+  if (blob) {
+#ifdef _WIN32
+    free(blob);
+#else
+    munmap(blob, blobinfo.blob_size);
+#endif
+  }
+}
 
-  // Offset the pointers in the table using the base mmap address.
-  moddef = blob;
-  while (moddef->name) {
-    moddef->name = (char *)((uintptr_t)moddef->name + (uintptr_t)blob);
-    if (moddef->code != 0) {
-      moddef->code = (unsigned char *)((uintptr_t)moddef->code + (uintptr_t)blob);
+/**
+ * Main entry point to deploy-stub.
+ */
+#if defined(_WIN32) && PY_MAJOR_VERSION >= 3
+int wmain(int argc, wchar_t *argv[]) {
+#else
+int main(int argc, char *argv[]) {
+#endif
+  int retval;
+  struct _frozen *moddef;
+  void *blob = NULL;
+
+  /*
+  printf("blob_offset: %d\n", (int)blobinfo.blob_offset);
+  printf("blob_size: %d\n", (int)blobinfo.blob_size);
+  printf("version: %d\n", (int)blobinfo.version);
+  printf("num_pointers: %d\n", (int)blobinfo.num_pointers);
+  printf("codepage: %d\n", (int)blobinfo.codepage);
+  printf("flags: %d\n", (int)blobinfo.flags);
+  printf("reserved: %d\n", (int)blobinfo.reserved);
+  */
+
+  // If we have a blob offset, we have to map the blob to memory.
+  if (blobinfo.version == 0 || blobinfo.blob_offset != 0) {
+    void *blob = map_blob((off_t)blobinfo.blob_offset, (size_t)blobinfo.blob_size);
+    assert(blob != NULL);
+
+    // Offset the pointers in the header using the base mmap address.
+    if (blobinfo.version > 0 && blobinfo.num_pointers > 0) {
+      uint32_t i;
+      for (i = 0; i < blobinfo.num_pointers; ++i) {
+        if (i == 0 || blobinfo.pointers[i] != 0) {
+          blobinfo.pointers[i] = (void *)((uintptr_t)blobinfo.pointers[i] + (uintptr_t)blob);
+        }
+      }
+    } else {
+      blobinfo.pointers[0] = blob;
+    }
+
+    // Offset the pointers in the module table using the base mmap address.
+    moddef = blobinfo.pointers[0];
+    while (moddef->name) {
+      moddef->name = (char *)((uintptr_t)moddef->name + (uintptr_t)blob);
+      if (moddef->code != 0) {
+        moddef->code = (unsigned char *)((uintptr_t)moddef->code + (uintptr_t)blob);
+      }
+      //printf("MOD: %s %p %d\n", moddef->name, (void*)moddef->code, moddef->size);
+      moddef++;
     }
-    //printf("MOD: %s %p %d\n", moddef->name, (void*)moddef->code, moddef->size);
-    moddef++;
   }
 
+#ifdef _WIN32
+  if (codepage != 0) {
+    SetConsoleCP(codepage);
+    SetConsoleOutputCP(codepage);
+  }
+#endif
+
   // Run frozen application
-  PyImport_FrozenModules = blob;
+  PyImport_FrozenModules = blobinfo.pointers[0];
   retval = Py_FrozenMain(argc, argv);
 
-  // Free resources
-#ifdef _WIN32
-  free(blob);
-#else
-  munmap(blob, size);
-#endif
+  unmap_blob(blob);
   return retval;
 }