Merge pull request #2248 from BDisp/non-bmp-fix-2247

Fixes #2247. Preparing for the NStack v1.0.7 which now handling properly non-BMP code points.

Terminal.Gui/ConsoleDrivers/FakeDriver/FakeConsole.cs

@@ -1401,7 +1401,10 @@ namespace Terminal.Gui {
 		/// <param name="buffer"></param>
 		public static void Write (char [] buffer)
 		{
-			throw new NotImplementedException ();
+			_buffer [CursorLeft, CursorTop] = (char)0;
+			foreach (var ch in buffer) {
+				_buffer [CursorLeft, CursorTop] += ch;
+			}
 		}
 
 		//

Terminal.Gui/ConsoleDrivers/FakeDriver/FakeDriver.cs

@@ -104,12 +104,12 @@ namespace Terminal.Gui {
 					needMove = false;
 				}
 				if (runeWidth < 2 && ccol > 0
-					&& Rune.ColumnWidth ((char)contents [crow, ccol - 1, 0]) > 1) {
+					&& Rune.ColumnWidth ((Rune)contents [crow, ccol - 1, 0]) > 1) {
 
 					contents [crow, ccol - 1, 0] = (int)(uint)' ';
 
 				} else if (runeWidth < 2 && ccol <= Clip.Right - 1
-					&& Rune.ColumnWidth ((char)contents [crow, ccol, 0]) > 1) {
+					&& Rune.ColumnWidth ((Rune)contents [crow, ccol, 0]) > 1) {
 
 					contents [crow, ccol + 1, 0] = (int)(uint)' ';
 					contents [crow, ccol + 1, 2] = 1;
@@ -234,7 +234,12 @@ namespace Terminal.Gui {
 						if (color != redrawColor)
 							SetColor (color);
 
-						FakeConsole.Write ((char)contents [row, col, 0]);
+						Rune rune = contents [row, col, 0];
+						if (Rune.DecodeSurrogatePair (rune, out char [] spair)) {
+							FakeConsole.Write (spair);
+						} else {
+							FakeConsole.Write ((char)rune);
+						}
 						contents [row, col, 2] = 0;
 					}
 				}

Terminal.Gui/Core/ConsoleDriver.cs

@@ -681,7 +681,7 @@ namespace Terminal.Gui {
 		/// <param name="col">Column to move the cursor to.</param>
 		/// <param name="row">Row to move the cursor to.</param>
 		public abstract void Move (int col, int row);
-		
+
 		/// <summary>
 		/// Adds the specified rune to the display at the current cursor position.
 		/// </summary>
@@ -696,11 +696,10 @@ namespace Terminal.Gui {
 		/// <returns></returns>
 		public static Rune MakePrintable (Rune c)
 		{
-			var controlChars = c & 0xFFFF;
-			if (controlChars <= 0x1F || controlChars >= 0X7F && controlChars <= 0x9F) {
+			if (c <= 0x1F || (c >= 0X7F && c <= 0x9F)) {
 				// ASCII (C0) control characters.
 				// C1 control characters (https://www.aivosto.com/articles/control-characters.html#c1)
-				return new Rune (controlChars + 0x2400);
+				return new Rune (c + 0x2400);
 			}
 
 			return c;

Terminal.Gui/Core/TextFormatter.cs

@@ -293,12 +293,6 @@ namespace Terminal.Gui {
 			}
 		}
 
-		/// <summary>
-		/// Specifies the mask to apply to the hotkey to tag it as the hotkey. The default value of <c>0x100000</c> causes
-		/// the underlying Rune to be identified as a "private use" Unicode character.
-		/// </summary>HotKeyTagMask
-		public uint HotKeyTagMask { get; set; } = 0x100000;
-
 		/// <summary>
 		/// Gets the cursor position from <see cref="HotKey"/>. If the <see cref="HotKey"/> is defined, the cursor will be positioned over it.
 		/// </summary>
@@ -317,8 +311,9 @@ namespace Terminal.Gui {
 			get {
 				// With this check, we protect against subclasses with overrides of Text
 				if (ustring.IsNullOrEmpty (Text) || Size.IsEmpty) {
-					lines = new List<ustring> ();
-					lines.Add (ustring.Empty);
+					lines = new List<ustring> {
+						ustring.Empty
+					};
 					NeedsFormat = false;
 					return lines;
 				}
@@ -716,7 +711,7 @@ namespace Terminal.Gui {
 		}
 
 		static char [] whitespace = new char [] { ' ', '\t' };
-		private int hotKeyPos;
+		private int hotKeyPos = -1;
 
 		/// <summary>
 		/// Reformats text into lines, applying text alignment and optionally wrapping text to new lines on word boundaries.
@@ -1113,14 +1108,13 @@ namespace Terminal.Gui {
 		/// <returns>The text with the hotkey tagged.</returns>
 		/// <remarks>
 		/// The returned string will not render correctly without first un-doing the tag. To undo the tag, search for 
-		/// Runes with a bitmask of <c>otKeyTagMask</c> and remove that bitmask.
 		/// </remarks>
 		public ustring ReplaceHotKeyWithTag (ustring text, int hotPos)
 		{
 			// Set the high bit
 			var runes = text.ToRuneList ();
 			if (Rune.IsLetterOrNumber (runes [hotPos])) {
-				runes [hotPos] = new Rune ((uint)runes [hotPos] | HotKeyTagMask);
+				runes [hotPos] = new Rune ((uint)runes [hotPos]);
 			}
 			return ustring.Make (runes);
 		}
@@ -1297,13 +1291,13 @@ namespace Terminal.Gui {
 							rune = runes [idx];
 						}
 					}
-					if ((rune & HotKeyTagMask) == HotKeyTagMask) {
+					if (idx == HotKeyPos) {
 						if ((isVertical && textVerticalAlignment == VerticalTextAlignment.Justified) ||
-						    (!isVertical && textAlignment == TextAlignment.Justified)) {
+						(!isVertical && textAlignment == TextAlignment.Justified)) {
 							CursorPosition = idx - start;
 						}
 						Application.Driver?.SetAttribute (hotColor);
-						Application.Driver?.AddRune ((Rune)((uint)rune & ~HotKeyTagMask));
+						Application.Driver?.AddRune (rune);
 						Application.Driver?.SetAttribute (normalColor);
 					} else {
 						Application.Driver?.AddRune (rune);

UnitTests/ConsoleDriverTests.cs

@@ -614,7 +614,7 @@ namespace Terminal.Gui.ConsoleDrivers {
 		[InlineData (0x0000001F, 0x241F)]
 		[InlineData (0x0000007F, 0x247F)]
 		[InlineData (0x0000009F, 0x249F)]
-		[InlineData (0x0001001A, 0x241A)]
+		[InlineData (0x0001001A, 0x1001A)]
 		public void MakePrintable_Converts_Control_Chars_To_Proper_Unicode (uint code, uint expected)
 		{
 			var actual = ConsoleDriver.MakePrintable (code);

UnitTests/TestHelpers.cs

@@ -53,7 +53,15 @@ class TestHelpers {
 
 		for (int r = 0; r < driver.Rows; r++) {
 			for (int c = 0; c < driver.Cols; c++) {
-				sb.Append ((char)contents [r, c, 0]);
+				Rune rune = contents [r, c, 0];
+				if (Rune.DecodeSurrogatePair (rune, out char [] spair)) {
+					sb.Append (spair);
+				} else {
+					sb.Append ((char)rune);
+				}
+				if (Rune.ColumnWidth (rune) > 1) {
+					c++;
+				}
 			}
 			sb.AppendLine ();
 		}
@@ -82,7 +90,7 @@ class TestHelpers {
 
 	public static Rect AssertDriverContentsWithFrameAre (string expectedLook, ITestOutputHelper output)
 	{
-		var lines = new List<List<char>> ();
+		var lines = new List<List<Rune>> ();
 		var sb = new StringBuilder ();
 		var driver = ((FakeDriver)Application.Driver);
 		var x = -1;
@@ -93,15 +101,15 @@ class TestHelpers {
 		var contents = driver.Contents;
 
 		for (int r = 0; r < driver.Rows; r++) {
-			var runes = new List<char> ();
+			var runes = new List<Rune> ();
 			for (int c = 0; c < driver.Cols; c++) {
-				var rune = (char)contents [r, c, 0];
+				var rune = (Rune)contents [r, c, 0];
 				if (rune != ' ') {
 					if (x == -1) {
 						x = c;
 						y = r;
 						for (int i = 0; i < c; i++) {
-							runes.InsertRange (i, new List<char> () { ' ' });
+							runes.InsertRange (i, new List<Rune> () { ' ' });
 						}
 					}
 					if (Rune.ColumnWidth (rune) > 1) {
@@ -130,7 +138,7 @@ class TestHelpers {
 
 		// Remove trailing whitespace on each line
 		for (int r = 0; r < lines.Count; r++) {
-			List<char> row = lines [r];
+			List<Rune> row = lines [r];
 			for (int c = row.Count - 1; c >= 0; c--) {
 				var rune = row [c];
 				if (rune != ' ' || (row.Sum (x => Rune.ColumnWidth (x)) == w)) {
@@ -140,9 +148,9 @@ class TestHelpers {
 			}
 		}
 
-		// Convert char list to string
+		// Convert Rune list to string
 		for (int r = 0; r < lines.Count; r++) {
-			var line = new string (lines [r].ToArray ());
+			var line = NStack.ustring.Make (lines [r]).ToString ();
 			if (r == lines.Count - 1) {
 				sb.Append (line);
 			} else {

UnitTests/TextFormatterTests.cs

@@ -2424,38 +2424,38 @@ namespace Terminal.Gui.Core {
 			var tf = new TextFormatter ();
 			ustring text = "test";
 			int hotPos = 0;
-			uint tag = tf.HotKeyTagMask | 't';
+			uint tag = 't';
 
 			Assert.Equal (ustring.Make (new Rune [] { tag, 'e', 's', 't' }), tf.ReplaceHotKeyWithTag (text, hotPos));
 
-			tag = tf.HotKeyTagMask | 'e';
+			tag = 'e';
 			hotPos = 1;
 			Assert.Equal (ustring.Make (new Rune [] { 't', tag, 's', 't' }), tf.ReplaceHotKeyWithTag (text, hotPos));
 
 			var result = tf.ReplaceHotKeyWithTag (text, hotPos);
-			Assert.Equal ('e', (uint)(result.ToRunes () [1] & ~tf.HotKeyTagMask));
+			Assert.Equal ('e', (uint)(result.ToRunes () [1]));
 
 			text = "Ok";
-			tag = 0x100000 | 'O';
+			tag = 'O';
 			hotPos = 0;
 			Assert.Equal (ustring.Make (new Rune [] { tag, 'k' }), result = tf.ReplaceHotKeyWithTag (text, hotPos));
-			Assert.Equal ('O', (uint)(result.ToRunes () [0] & ~tf.HotKeyTagMask));
+			Assert.Equal ('O', (uint)(result.ToRunes () [0]));
 
 			text = "[◦ Ok ◦]";
 			text = ustring.Make (new Rune [] { '[', '◦', ' ', 'O', 'k', ' ', '◦', ']' });
 			var runes = text.ToRuneList ();
 			Assert.Equal (text.RuneCount, runes.Count);
 			Assert.Equal (text, ustring.Make (runes));
-			tag = tf.HotKeyTagMask | 'O';
+			tag = 'O';
 			hotPos = 3;
 			Assert.Equal (ustring.Make (new Rune [] { '[', '◦', ' ', tag, 'k', ' ', '◦', ']' }), result = tf.ReplaceHotKeyWithTag (text, hotPos));
-			Assert.Equal ('O', (uint)(result.ToRunes () [3] & ~tf.HotKeyTagMask));
+			Assert.Equal ('O', (uint)(result.ToRunes () [3]));
 
 			text = "^k";
 			tag = '^';
 			hotPos = 0;
 			Assert.Equal (ustring.Make (new Rune [] { tag, 'k' }), result = tf.ReplaceHotKeyWithTag (text, hotPos));
-			Assert.Equal ('^', (uint)(result.ToRunes () [0] & ~tf.HotKeyTagMask));
+			Assert.Equal ('^', (uint)(result.ToRunes () [0]));
 		}
 
 		[Fact]
@@ -4163,5 +4163,101 @@ This TextFormatter (tf2) is rewritten.
 			Assert.Equal ("你", ((Rune)usToRunes [9]).ToString ());
 			Assert.Equal ("你", s [9].ToString ());
 		}
+
+		[Fact, AutoInitShutdown]
+		public void Non_Bmp_ConsoleWidth_ColumnWidth_Equal_Two ()
+		{
+			ustring us = "\U0001d539";
+			Rune r = 0x1d539;
+
+			Assert.Equal ("𝔹", us);
+			Assert.Equal ("𝔹", r.ToString ());
+			Assert.Equal (us, r.ToString ());
+
+			Assert.Equal (2, us.ConsoleWidth);
+			Assert.Equal (2, Rune.ColumnWidth (r));
+
+			var win = new Window (us);
+			var label = new Label (ustring.Make (r));
+			var tf = new TextField (us) { Y = 1, Width = 3 };
+			win.Add (label, tf);
+			var top = Application.Top;
+			top.Add (win);
+
+			Application.Begin (top);
+			((FakeDriver)Application.Driver).SetBufferSize (10, 4);
+
+			var expected = @"
+┌ 𝔹 ────┐
+│𝔹      │
+│𝔹      │
+└────────┘";
+			TestHelpers.AssertDriverContentsWithFrameAre (expected, output);
+
+			TestHelpers.AssertDriverContentsAre (expected, output);
+
+			var expectedColors = new Attribute [] {
+				// 0
+				Colors.Base.Normal,
+				// 1
+				Colors.Base.Focus,
+				// 2
+				Colors.Base.HotNormal
+			};
+
+			TestHelpers.AssertDriverColorsAre (@"
+0222200000
+0000000000
+0111000000
+0000000000", expectedColors);
+		}
+
+		[Fact, AutoInitShutdown]
+		public void CJK_Compatibility_Ideographs_ConsoleWidth_ColumnWidth_Equal_Two ()
+		{
+			ustring us = "\U0000f900";
+			Rune r = 0xf900;
+
+			Assert.Equal ("豈", us);
+			Assert.Equal ("豈", r.ToString ());
+			Assert.Equal (us, r.ToString ());
+
+			Assert.Equal (2, us.ConsoleWidth);
+			Assert.Equal (2, Rune.ColumnWidth (r));
+
+			var win = new Window (us);
+			var label = new Label (ustring.Make (r));
+			var tf = new TextField (us) { Y = 1, Width = 3 };
+			win.Add (label, tf);
+			var top = Application.Top;
+			top.Add (win);
+
+			Application.Begin (top);
+			((FakeDriver)Application.Driver).SetBufferSize (10, 4);
+
+			var expected = @"
+┌ 豈 ────┐
+│豈      │
+│豈      │
+└────────┘";
+			TestHelpers.AssertDriverContentsWithFrameAre (expected, output);
+
+			TestHelpers.AssertDriverContentsAre (expected, output);
+
+			var expectedColors = new Attribute [] {
+				// 0
+				Colors.Base.Normal,
+				// 1
+				Colors.Base.Focus,
+				// 2
+				Colors.Base.HotNormal
+			};
+
+			TestHelpers.AssertDriverColorsAre (@"
+0222200000
+0000000000
+0111000000
+0000000000", expectedColors);
+		}
 	}
 }