Regression suite: 2.7 compatibility, drop old logic to find assimp binary and require it to be explicitly specified. Process files in a stable order to make results diffable.

test/regression/gen_db.py

@@ -40,8 +40,15 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 # ---------------------------------------------------------------------------
 
-"""Generate the regression database db.zip from the files in the <root>
-/test/models directory. Older databases are overwritten with no prompt.
+"""
+Generate the regression database db.zip from the files in the <root>/test/models
+directory. Older databases are overwritten with no prompt but can be restored
+using Git as needed.
+
+Use --help for usage.
+
+On Windows, use ``py run.py <arguments>`` to make sure command line parameters
+are forwarded to the script.
 """
 
 import sys
@@ -52,9 +59,14 @@ import zipfile
 import settings
 import utils
 
-usage = """gen_db [-i=...] [-e=...] [-p] [-n]
+usage = """gen_db [assimp_binary] [-i=...] [-e=...] [-p] [-n]
+
+The assimp_cmd (or assimp) binary to use is specified by the first
+command line argument and defaults to ``assimp``.
+
+To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating
+configs for an IDE, make sure to build the assimp_cmd project.
 
-(lists of file extensions are comma delimited, i.e. `3ds,lwo,x`)
 -i,--include: List of file extensions to update dumps for. If omitted,
          all file extensions are updated except those in `exclude`.
 
@@ -66,6 +78,8 @@ usage = """gen_db [-i=...] [-e=...] [-p] [-n]
          Dont' change anything.
 
 -n,--nozip: Don't pack to ZIP archive. Keep all dumps in individual files.
+
+(lists of file extensions are comma delimited, i.e. `3ds,lwo,x`)
 """
 
 # -------------------------------------------------------------------------------
@@ -87,7 +101,7 @@ def process_dir(d, outfile, file_filter):
                 outf = os.path.join(os.getcwd(), settings.database_name,
                     utils.hashing(fullp, pp))
 
-                cmd = [utils.assimp_bin_path,"dump",fullp,outf,"-b","-s","-l"] + pp.split()
+                cmd = [ assimp_bin_path, "dump", fullp, outf, "-b", "-s", "-l" ] + pp.split()
                 outfile.write("assimp dump "+"-"*80+"\n")
                 outfile.flush()
                 if subprocess.call(cmd, stdout=outfile, stderr=outfile, shell=False):
@@ -158,7 +172,8 @@ def gen_db(ext_list,outfile):
 
 # -------------------------------------------------------------------------------
 if __name__ == "__main__":
-    utils.find_assimp_or_die()
+    assimp_bin_path = sys.argv[1] if len(sys.argv) > 1 else 'assimp'
+
     def clean(f):
         f = f.strip("* \'")
         return "."+f if f[:1] != '.' else f
@@ -184,7 +199,7 @@ if __name__ == "__main__":
             
     outfile = open(os.path.join("..", "results", "gen_regression_db_output.txt"), "w")
     if ext_list is None:
-        (ext_list, err) = subprocess.Popen([utils.assimp_bin_path, "listext"],
+        (ext_list, err) = subprocess.Popen([assimp_bin_path, "listext"],
             stdout=subprocess.PIPE).communicate()
         ext_list = str(ext_list).lower().split(";")
 

test/regression/run.py

@@ -41,8 +41,16 @@
 # ---------------------------------------------------------------------------
 
 """
-Run the regression test suite using the settings from settings.py.
+Run the regression test suite using settings from settings.py.
 
+The assimp_cmd (or assimp) binary to use is specified by the first
+command line argument and defaults to ``assimp``.
+
+To build, set ``ASSIMP_BUILD_ASSIMP_TOOLS=ON`` in CMake. If generating
+configs for an IDE, make sure to build the assimp_cmd project.
+
+On Windows, use ``py run.py <path to assimp>`` to make sure the command
+line parameter is forwarded to the script.
 """
 
 import sys
@@ -124,8 +132,11 @@ class results:
     def report_results(self):
         """Write results to ../results/run_regression_suite_failures.txt"""
 
+        count_success = len(self.success)
+        count_fail = len(self.failures)
+        percent_good = float(count_success) / (count_success + count_fail)
         print("\n" + ('='*60) + "\n" + "SUCCESS: {0}\nFAILURE: {1}\nPercentage good: {2}".format(
-            len(self.success), len(self.failures), len(self.success)/(len(self.success)+len(self.failures))  ) + 
+            count_success, count_fail, percent_good) + 
               "\n" + ('='*60) + "\n")
 
         with open(os.path.join('..', 'results',outfilename_failur), "wt") as f:
@@ -138,7 +149,7 @@ class results:
                  + " for more details\n\n") 
     
 # -------------------------------------------------------------------------------
-def mkoutputdir_andgetpath(fullpath, myhash, app):
+def prepare_output_dir(fullpath, myhash, app):
     outfile = os.path.join(settings.results, "tmp", os.path.split(fullpath)[1] + "_" + myhash)
     try:
         os.mkdir(outfile)
@@ -154,7 +165,7 @@ def process_dir(d, outfile_results, zipin, result):
     shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}
 
     print("Processing directory " + d)
-    for f in os.listdir(d):
+    for f in sorted(os.listdir(d)):
         fullpath = os.path.join(d, f)
         if os.path.isdir(fullpath) and not f == ".svn":
             process_dir(fullpath, outfile_results, zipin, result)
@@ -167,13 +178,16 @@ def process_dir(d, outfile_results, zipin, result):
         for pppreset in settings.pp_configs_to_test:
             filehash = utils.hashing(fullpath, pppreset)
             failure = False
+
             try:
                 input_expected = zipin.open(filehash, "r").read()
                 # empty dump files indicate 'expected import failure'
                 if not len(input_expected):
                    failure = True
             except KeyError:
-                #print("Didn't find "+fullpath+" (Hash is "+filehash+") in database")
+                # TODO(acgessler): Keep track of this and report as error in the end.
+                print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\
+                    "regression database? Use gen_db.zip to re-generate.")
                 continue
 
             # Ignore extensions via settings.py configured list
@@ -184,13 +198,18 @@ def process_dir(d, outfile_results, zipin, result):
 
             print("-"*60 + "\n  " + os.path.realpath(fullpath) + " pp: " + pppreset) 
             
-            outfile_actual = mkoutputdir_andgetpath(fullpath, filehash, "ACTUAL")
-            outfile_expect = mkoutputdir_andgetpath(fullpath, filehash, "EXPECT")
+            outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL")
+            outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT")
             outfile_results.write("assimp dump    "+"-"*80+"\n")
             outfile_results.flush()
 
-            command = [utils.assimp_bin_path,"dump",fullpath,outfile_actual,"-b","-s","-l"]+pppreset.split()
+            command = [assimp_bin_path,
+                "dump",
+                fullpath, outfile_actual, "-b", "-s", "-l" ] +\
+                pppreset.split()
+
             r = subprocess.call(command, **shellparams)
+            print(r)
 
             if r and not failure:
                 result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
@@ -216,7 +235,7 @@ def process_dir(d, outfile_results, zipin, result):
             outfile_results.write("assimp cmpdump "+"-"*80+"\n")
             outfile_results.flush()
 
-            command = [utils.assimp_bin_path,'cmpdump',outfile_actual,outfile_expect]
+            command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ]
             if subprocess.call(command, **shellparams) != 0:
                 result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
                 continue 
@@ -235,7 +254,6 @@ def del_folder_with_contents(folder):
 
 # -------------------------------------------------------------------------------
 def run_test():
-    utils.find_assimp_or_die()
     tmp_target_path = os.path.join(settings.results, "tmp")
     try: 
         os.mkdir(tmp_target_path) 
@@ -261,6 +279,8 @@ def run_test():
 
 # -------------------------------------------------------------------------------
 if __name__ == "__main__":
+    assimp_bin_path = sys.argv[1] if len(sys.argv) > 1 else 'assimp'
+    print('Using assimp binary: ' + assimp_bin_path)
     run_test()
 
 # vim: ai ts=4 sts=4 et sw=4

test/regression/utils.py

@@ -40,7 +40,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 # ---------------------------------------------------------------------------
 
-"""Shared stuff for the gen_db and run scripts """
+"""Shared stuff for the gen_db and run scripts"""
 
 # -------------------------------------------------------------------------------
 def hashing(file,pp):
@@ -51,75 +51,14 @@ def hashing(file,pp):
     and platforms, so we implement the hashing manually.
     """
 
-    def myhash(instring):
-        # sdbm hash
-        res = 0
-        for t in instring:
-            res = (ord(t) + (res<<6) + (res<<16) - res) % 2**32
-        return res
+    file = file.replace('\\','/')+":"+pp
+    # SDBM hash
+    res = 0
+    for t in file:
+        res = (ord(t) + (res<<6) + (res<<16) - res) % 2**32
 
-    return hex(myhash(file.replace('\\','/')+":"+pp))
+    # Python 2.7 normalization: strip 'L' suffix.
+    return hex(res).rstrip('L')
 
 
-assimp_bin_path = None
-# -------------------------------------------------------------------------------
-def find_assimp_or_die():
-    """Find assimp_cmd's binary for the current platform.
-    
-    The path to the binary is stored in assimp_bin_path, the process
-    is aborted if it can't be found.
-
-    """
-
-    import os
-    import platform
-    import sys
-
-    def locate_file(f_list):
-        for f in f_list:
-            try:
-                fl = open(f,"rb")
-            except IOError:
-                continue
-            fl.close()
-            return f
-        return None
-
-    global assimp_bin_path
-    if os.name == "nt":
-        search_x86 = [
-            os.path.join("assimp.exe"),
-            os.path.join("..","..","bin","assimpcmd_release-dll_Win32","assimp.exe"),
-            os.path.join("..","..","bin","x86","assimp"),
-            os.path.join("..","..","bin","Release","assimp.exe")
-        ]
-        if platform.machine() == "x86":
-            search = search_x86
-        else: # amd64, hopefully
-            search = [
-                os.path.join("..","..","bin","assimpcmd_release-dll_x64","assimp.exe"),
-                os.path.join("..","..","bin","x64","assimp")
-            ]
-            # x64 platform does not guarantee a x64 build. Also look for x86 as last paths.
-            search += search_x86
-        
-        assimp_bin_path = locate_file(search)
-        if assimp_bin_path is None:
-            print("Can't locate assimp_cmd binary")
-            print("Looked in", search)
-            sys.exit(-5)
-
-        print("Located assimp/assimp_cmd binary from", assimp_bin_path)
-    elif os.name == "posix":
-        #search = [os.path.join("..","..","bin","gcc","assimp"),
-        #    os.path.join("/usr","local","bin",'assimp')]
-        assimp_bin_path = "assimp"
-        print("Taking system-wide assimp binary")
-    else:
-        print("Unsupported operating system")
-        sys.exit(-5)
-
-if __name__ == '__main__':
-    find_assimp_or_die()
-
  # vim: ai ts=4 sts=4 et sw=4