Fix #9199 (#9262)

* add TEST_INVALID_UNICODE_FS to Unicode sys tests

* change Azure CI to 10.14, disable invalid Unicode filenames on that target

* also disable U+1FFFF

azure-pipelines.yml

@@ -118,7 +118,7 @@ stages:
       - job: TestMac
         dependsOn: BuildMac
         pool:
-          vmImage: 'macOS-10.13'
+          vmImage: 'macOS-10.14'
         strategy:
           matrix:
             macro:
@@ -174,7 +174,10 @@ stages:
           - script: brew install $BREW_PACKAGES
             condition: and(succeeded(), variables['BREW_PACKAGES'])
             displayName: Install homebrew packages
-          - script: haxe RunCi.hxml
+          - script: |
+              # disable invalid Unicode filenames on APFS
+              echo "" > sys/compile-fs.hxml
+              haxe RunCi.hxml
             workingDirectory: $(Build.SourcesDirectory)/tests
             displayName: Test
 

tests/sys/compile-each.hxml

@@ -8,4 +8,5 @@
 -D source-header=''
 --debug
 -lib utest
--p src
+-p src
+compile-fs.hxml

tests/sys/compile-fs.hxml

@@ -0,0 +1,4 @@
+# comment the following line to disable testing the filesystem with invalid
+# Unicode codepoints; these will not work on APFS
+
+-D TEST_INVALID_UNICODE_FS

tests/sys/genTestRes.py

@@ -4,8 +4,18 @@
 # The test vector printf'ed into data.bin, as well as the names in filenames()
 # should correspond exactly to the sequences in UnicodeSequences.valid.
 
+# Run with:
+#   python3 genTestRes.py
+# Or:
+#   python3 genTestRes.py TEST_INVALID_UNICODE_FS
+# The latter will attempt to create filenames which contain invalid Unicode
+# codepoints; this does not work on some filesystems, e.g. APFS.
+
 import os
 import shutil
+import sys
+
+MODE = " ".join(sys.argv[1:])
 
 TESTDIR = "test-res"
 
@@ -17,19 +27,19 @@ os.mkdir(TESTDIR)
 
 # Unicode test vectors
 allUnicode = [
-    [0x01],
+    [0x01], # will not work on NTFS
     [0x7F],
     [0xC2, 0x80],
     [0xDF, 0xBF],
     [0xE0, 0xA0, 0x80],
-    [0xED, 0x9F, 0xBF],
+    [0xED, 0x9F, 0xBF], # will not work on APFS
     [0xEE, 0x80, 0x80],
     [0xEF, 0xBF, 0xBD],
     [0xF0, 0x90, 0x80, 0x80],
-    [0xF0, 0x9F, 0xBF, 0xBF],
-    [0xF3, 0xBF, 0xBF, 0xBF],
+    [0xF0, 0x9F, 0xBF, 0xBF], # will not work on APFS
+    [0xF3, 0xBF, 0xBF, 0xBF], # will not work on APFS
     [0xF4, 0x80, 0x80, 0x80],
-    [0xF4, 0x8F, 0xBF, 0xBF],
+    [0xF4, 0x8F, 0xBF, 0xBF], # will not work on APFS
     [0xF0, 0x9F, 0x98, 0x82, 0xF0, 0x9F, 0x98, 0x84, 0xF0, 0x9F, 0x98, 0x99],
     [0xC8, 0xA7],
     [0xE4, 0xB8, 0xAD, 0xE6, 0x96, 0x87, 0xEF, 0xBC, 0x8C, 0xE3, 0x81, 0xAB, 0xE3, 0x81, 0xBB, 0xE3, 0x82, 0x93, 0xE3, 0x81, 0x94]
@@ -43,6 +53,13 @@ allFilenames = allStrings[:]
 if os.name == "nt":
   allFilenames.remove(bytes([0x01]).decode("utf-8"))
 
+# on APFS (macOS 10.13+), filenames must consist of valid Unicode codepoints
+if MODE != "TEST_INVALID_UNICODE_FS":
+  allFilenames.remove(bytes([0xED, 0x9F, 0xBF]).decode("utf-8"))
+  allFilenames.remove(bytes([0xF0, 0x9F, 0xBF, 0xBF]).decode("utf-8"))
+  allFilenames.remove(bytes([0xF3, 0xBF, 0xBF, 0xBF]).decode("utf-8"))
+  allFilenames.remove(bytes([0xF4, 0x8F, 0xBF, 0xBF]).decode("utf-8"))
+
 allBinary = b""
 for data in allUnicode:
   allBinary += bytes(data) + b"\n"

tests/sys/src/TestUnicode.hx

@@ -58,7 +58,7 @@ class TestUnicode extends utest.Test {
 	];
 
 	// list of expected filenames in sub-directories
-	static var names:Array<UnicodeString> = (Sys.systemName() == "Windows" ? UnicodeSequences.valid.slice(1) : UnicodeSequences.valid);
+	static var names:Array<UnicodeString> = UnicodeSequences.validFilenames;
 
 	// extra files only present in the root test-res directory
 	static var namesRoot = names.concat([
@@ -132,7 +132,11 @@ class TestUnicode extends utest.Test {
 
 	function setupClass() {
 		FileSystem.createDirectory("temp-unicode");
+		#if TEST_INVALID_UNICODE_FS
+		Sys.command("python3", ["genTestRes.py", "TEST_INVALID_UNICODE_FS"]);
+		#else
 		Sys.command("python3", ["genTestRes.py"]);
+		#end
 	}
 
 	function teardownClass() {

tests/sys/src/UnicodeSequences.hx

@@ -39,6 +39,19 @@ class UnicodeSequences {
 		.concat([Only([0x1F602, 0x1F604, 0x1F619])]) // important (non-BMP) emoji
 		.concat(normal);
 
+	public static var validFilenames:Array<UnicodeString> = {
+		var valid = valid.copy();
+		if (Sys.systemName() == "Windows") valid = valid.filter(f -> !f.match(Only([0x0001])));
+		#if !(TEST_INVALID_UNICODE_FS)
+		valid = valid.filter(f ->
+			!f.match(Only([0xD7FF]))
+			&& !f.match(Only([0x1FFFF]))
+			&& !f.match(Only([0xFFFFF]))
+			&& !f.match(Only([0x10FFFF])));
+		#end
+		valid;
+	};
+
 	public static var validBytes = haxe.io.Bytes.ofHex(
 			"010A" +
 			"7F0A" +
@@ -76,12 +89,6 @@ class UnicodeSequences {
 		"\u0227\n" +
 		"\u4E2D\u6587\uFF0C\u306B\u307B\u3093\u3054\n";
 
-	// invalid sequences
-	public static var invalid:Array<UnicodeString> = [
-		Only([0xFFFE]),
-		Only([0xFFFF])
-	];
-
 	// utility methods
 
 	public static function unicodeCodepoints(str:String):Array<Int> {