makewheel: Support deterministic wheel creation via SOURCE_DATE_EPOCH

Setting SOURCE_DATE_EPOCH to a UNIX timestamp will clamp all dates to that, which enables bit-for-bit recreation of wheel files (assuming the sources are also bit-for-bit identical).  Furthermore, files are sorted when added, to avoid filesystem indeterminism.

makepanda/makewheel.py

@@ -9,6 +9,7 @@ import zipfile
 import hashlib
 import tempfile
 import subprocess
+import time
 from distutils.util import get_platform
 from distutils.sysconfig import get_config_var
 from optparse import OptionParser
@@ -301,6 +302,14 @@ class WheelFile(object):
         self.dep_paths = {}
         self.ignore_deps = set()
 
+        # This can be set if a reproducible (deterministic) build is desired, in
+        # which case we have to clamp all dates to the given SOURCE_DATE_EPOCH.
+        epoch = os.environ.get('SOURCE_DATE_EPOCH')
+        self.max_date_time = time.localtime(int(epoch) if epoch else time.time())[:6]
+        if self.max_date_time < (1980, 1, 1, 0, 0, 0):
+            # Earliest representable time in zip archives.
+            self.max_date_time = (1980, 1, 1, 0, 0, 0)
+
     def consider_add_dependency(self, target_path, dep, search_path=None):
         """Considers adding a dependency library.
         Returns the target_path if it was added, which may be different from
@@ -466,26 +475,29 @@ class WheelFile(object):
                 target_dep = os.path.dirname(target_path) + '/' + dep
                 self.consider_add_dependency(target_dep, dep)
 
-        # Calculate the SHA-256 hash and size.
-        sha = hashlib.sha256()
-        fp = open(source_path, 'rb')
+        if GetVerbose():
+            print("Adding {0} from {1}".format(target_path, orig_source_path))
+
+        zinfo = zipfile.ZipInfo.from_file(source_path, target_path)
+        if zinfo.date_time > self.max_date_time:
+            zinfo.date_time = self.max_date_time
+
+        # Copy the data to the zip file, while also calculating the SHA-256.
         size = 0
-        data = fp.read(1024 * 1024)
-        while data:
-            size += len(data)
-            sha.update(data)
-            data = fp.read(1024 * 1024)
-        fp.close()
+        sha = hashlib.sha256()
+        with open(source_path, 'rb') as source_fp, self.zip_file.open(zinfo, 'w') as target_fp:
+            data = source_fp.read(1024 * 1024)
+            while data:
+                size += len(data)
+                target_fp.write(data)
+                sha.update(data)
+                data = source_fp.read(1024 * 1024)
 
         # Save it in PEP-0376 format for writing out later.
         digest = urlsafe_b64encode(sha.digest()).decode('ascii')
         digest = digest.rstrip('=')
         self.records.append("{0},sha256={1},{2}\n".format(target_path, digest, size))
 
-        if GetVerbose():
-            print("Adding {0} from {1}".format(target_path, orig_source_path))
-        self.zip_file.write(source_path, target_path)
-
         #if temp:
         #    os.unlink(temp.name)
 
@@ -500,13 +512,19 @@ class WheelFile(object):
 
         if GetVerbose():
             print("Adding %s from data" % target_path)
-        self.zip_file.writestr(target_path, source_data)
+
+        zinfo = zipfile.ZipInfo(filename=target_path,
+                                date_time=self.max_date_time)
+        zinfo.compress_type = self.zip_file.compression
+        zinfo.external_attr = 0o600 << 16
+        self.zip_file.writestr(zinfo, source_data)
 
     def write_directory(self, target_dir, source_dir):
         """Adds the given directory recursively to the .whl file."""
 
         for root, dirs, files in os.walk(source_dir):
-            for file in files:
+            dirs.sort()
+            for file in sorted(files):
                 if os.path.splitext(file)[1] in EXCLUDE_EXT:
                     continue
 
@@ -520,7 +538,11 @@ class WheelFile(object):
         record_file = "{0}-{1}.dist-info/RECORD".format(self.name, self.version)
         self.records.append(record_file + ",,\n")
 
-        self.zip_file.writestr(record_file, "".join(self.records))
+        zinfo = zipfile.ZipInfo(filename=record_file,
+                                date_time=self.max_date_time)
+        zinfo.compress_type = self.zip_file.compression
+        zinfo.external_attr = 0o600 << 16
+        self.zip_file.writestr(zinfo, "".join(self.records))
         self.zip_file.close()
 
 
@@ -650,7 +672,7 @@ if __debug__:
     # Copy the extension modules from the panda3d directory.
     ext_suffix = GetExtensionSuffix()
 
-    for file in os.listdir(panda3d_dir):
+    for file in sorted(os.listdir(panda3d_dir)):
         if file == '__init__.py':
             pass
         elif file.endswith('.py') or (file.endswith(ext_suffix) and '.' not in file[:-len(ext_suffix)]):
@@ -668,7 +690,7 @@ if __debug__:
     # deploy_libs directory, for use by deploy-ng.
     ext_suffix = '.pyd' if sys.platform in ('win32', 'cygwin') else '.so'
 
-    for file in os.listdir(ext_mod_dir):
+    for file in sorted(os.listdir(ext_mod_dir)):
         if file.endswith(ext_suffix):
             source_path = os.path.join(ext_mod_dir, file)
 
@@ -704,14 +726,14 @@ if __debug__:
     whl.write_directory('panda3d/models', models_dir)
 
     # Add the pandac tree for backward compatibility.
-    for file in os.listdir(pandac_dir):
+    for file in sorted(os.listdir(pandac_dir)):
         if file.endswith('.py'):
             whl.write_file('pandac/' + file, os.path.join(pandac_dir, file))
 
     # Let's also add the interrogate databases.
     input_dir = os.path.join(pandac_dir, 'input')
     if os.path.isdir(input_dir):
-        for file in os.listdir(input_dir):
+        for file in sorted(os.listdir(input_dir)):
             if file.endswith('.in'):
                 whl.write_file('pandac/input/' + file, os.path.join(input_dir, file))
 
@@ -720,7 +742,7 @@ if __debug__:
     entry_points += 'eggcacher = direct.directscripts.eggcacher:main\n'
     entry_points += 'pfreeze = direct.dist.pfreeze:main\n'
     tools_init = ''
-    for file in os.listdir(bin_dir):
+    for file in sorted(os.listdir(bin_dir)):
         basename = os.path.splitext(file)[0]
         if basename in ('eggcacher', 'packpanda'):
             continue