Fixed UTF-8 offset issues.

maxguitextareascintilla.mod/common.bmx

@@ -70,7 +70,6 @@ Extern
 	Function bmx_mgta_scintilla_linefromposition:Int(handle:Byte Ptr, index:Int)
 	Function bmx_mgta_scintilla_appendtext(handle:Byte Ptr, Text:String)
 	Function bmx_mgta_scintilla_scrolltoend(handle:Byte Ptr)
-	Function bmx_mgta_scintilla_bytefromchar:Int(handle:Byte Ptr, charPos:Int, startBytePos:Int, startCharPos:Int)
 	Function bmx_mgta_scintilla_getselectionlength:Int(handle:Byte Ptr, units:Int)
 	Function bmx_mgta_scintilla_addtext(handle:Byte Ptr, Text:Byte Ptr)
 	Function bmx_mgta_scintilla_textwidth:Int(handle:Byte Ptr, Text:String)
@@ -78,6 +77,7 @@ Extern
 	Function bmx_mgta_scintilla_setmarginleft(handle:Byte Ptr, leftmargin:Int)
 	Function bmx_mgta_scintilla_setcaretwidth(handle:Byte Ptr, width:Int)
 	Function bmx_mgta_scintilla_setcaretcolor(handle:Byte Ptr, r:Int, g:Int, b:Int)
+	Function bmx_mgta_scintilla_startendfromchar(handle:Byte Ptr, pos:Int, length:Int, startPos:Int Var, endPos:Int Var)
 
 	Function bmx_mgta_scintilla_notifcation_update(obj:Object, handle:Byte Ptr)
 	

maxguitextareascintilla.mod/glue.c

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014-2018 Bruce A Henderson
+ Copyright (c) 2014-2019 Bruce A Henderson
  
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
@@ -153,48 +153,32 @@ int bmx_mgta_scintilla_charfrombyte(SCI_HANDLE sci, int pos, int startPos) {
 	return characterOffset;
 }
 
-/* startBytePos is a known byte offset for startCharPos */
-int bmx_mgta_scintilla_bytefromchar(SCI_HANDLE sci, int charLength, int startBytePos, int startCharPos) {
-	int i;
-	int characterOffset = startBytePos;
-
-	int characters = charLength;
-	if (!characters) {
-		return startBytePos;
-	}
-
-	int endBytePos = scintilla_send_message(sci, SCI_GETLENGTH, 0, 0);
-
-	if (characters == -1) {
-		return endBytePos;
+void bmx_mgta_scintilla_startendfromchar(SCI_HANDLE sci, int pos, int length, int * startPos, int * endPos) {
+	// all
+	if (length == -1) {
+		*startPos = 0;
+		*endPos = scintilla_send_message(sci, SCI_GETLENGTH, 0, 0);
+		return;
 	}
 
-	int length = endBytePos - startBytePos + 1;
+	int startLine = scintilla_send_message(sci, SCI_LINEFROMINDEXPOSITION, pos, SC_LINECHARACTERINDEX_UTF16);
+	int startLinePos = scintilla_send_message(sci, SCI_INDEXPOSITIONFROMLINE, startLine, SC_LINECHARACTERINDEX_UTF16);
 
-	struct Sci_TextRange range;
-	range.chrg.cpMin = startBytePos;
-	range.chrg.cpMax = endBytePos;
-	
-	range.lpstrText = malloc(length);
+	int actualStartLinePos = scintilla_send_message(sci, SCI_POSITIONFROMLINE, startLine, 0);
+	*startPos = scintilla_send_message(sci, SCI_POSITIONRELATIVECODEUNITS, actualStartLinePos, pos - startLinePos);
 	
-	int len = scintilla_send_message(sci, SCI_GETTEXTRANGE, 0, &range);
-
-	for (i = 0; i < len; i++) {
-		char c = range.lpstrText[i];
-		if ((c & 0xc0) != 0x80) {
-				if (! --characters) {
-					break;
-				}
+	if (length) {
+		*endPos = scintilla_send_message(sci, SCI_POSITIONRELATIVECODEUNITS, *startPos, length);
+		if (*endPos == 0) {
+			*endPos = scintilla_send_message(sci, SCI_GETLENGTH, 0, 0);
 		}
+	} else {
+		*endPos = *startPos;
 	}
-	free(range.lpstrText);
-
-	return startBytePos + i + 1;
 }
 
 int bmx_mgta_scintilla_positionfromline(SCI_HANDLE sci, int line, int valueInBytes) {
-	int bytePos = scintilla_send_message(sci, SCI_POSITIONFROMLINE, line, 0);
-	return (valueInBytes) ? bytePos : bmx_mgta_scintilla_charfrombyte(sci, bytePos, 0);
+	return scintilla_send_message(sci, SCI_INDEXPOSITIONFROMLINE, line, SC_LINECHARACTERINDEX_UTF16);
 }
 
 void bmx_mgta_scintilla_setselectionstart(SCI_HANDLE sci, int pos) {
@@ -278,15 +262,18 @@ int bmx_mgta_scintilla_getlength(SCI_HANDLE sci) {
 	return scintilla_send_message(sci, SCI_GETLENGTH, 0, 0);
 }
 
-int bmx_mgta_scintilla_getcurrentpos(SCI_HANDLE sci) {
-	int bytePos = scintilla_send_message(sci, SCI_GETSELECTIONSTART, 0, 0);
-	return bmx_mgta_scintilla_charfrombyte(sci, bytePos, 0);
-}
-
 int bmx_mgta_scintilla_getcurrentline(SCI_HANDLE sci) {
 	return scintilla_send_message(sci, SCI_LINEFROMPOSITION, scintilla_send_message(sci, SCI_GETSELECTIONSTART, 0, 0), 0);
 }
 
+int bmx_mgta_scintilla_getcurrentpos(SCI_HANDLE sci) {
+	int line = bmx_mgta_scintilla_getcurrentline(sci);
+	int startPos = scintilla_send_message(sci, SCI_POSITIONFROMLINE, line, 0);
+	int endPos = scintilla_send_message(sci, SCI_GETSELECTIONSTART, 0, 0);
+
+	return scintilla_send_message(sci, SCI_INDEXPOSITIONFROMLINE, line, SC_LINECHARACTERINDEX_UTF16) + scintilla_send_message(sci, SCI_COUNTCODEUNITS, startPos, endPos);
+}
+
 void bmx_mgta_scintilla_settabwidth(SCI_HANDLE sci, int tabs) {
 	scintilla_send_message(sci, SCI_SETTABWIDTH, tabs, 0);
 }
@@ -316,7 +303,7 @@ void bmx_mgta_scintilla_paste(SCI_HANDLE sci) {
 }
 
 int bmx_mgta_scintilla_linefromposition(SCI_HANDLE sci, int pos) {
-	return scintilla_send_message(sci, SCI_LINEFROMPOSITION, pos, 0);
+	return scintilla_send_message(sci, SCI_LINEFROMINDEXPOSITION, pos, SC_LINECHARACTERINDEX_UTF16);
 }
 
 void bmx_mgta_scintilla_appendtext(SCI_HANDLE sci, BBString * text) {
@@ -337,8 +324,9 @@ int bmx_mgta_scintilla_getselectionlength(SCI_HANDLE sci, int units) {
 		return endPos - startPos;
 	} else {
 		/* chars */
-		int startPos = bmx_mgta_scintilla_charfrombyte(sci, scintilla_send_message(sci, SCI_GETSELECTIONSTART, 0, 0), 0);
-		int length = bmx_mgta_scintilla_charfrombyte(sci, scintilla_send_message(sci, SCI_GETSELECTIONEND, 0, 0), startPos);
+		int startPos = scintilla_send_message(sci, SCI_GETSELECTIONSTART, 0, 0);
+		int endPos = scintilla_send_message(sci, SCI_GETSELECTIONEND, 0, 0);
+		int length = scintilla_send_message(sci, SCI_COUNTCODEUNITS, startPos, endPos);
 		return length;
 	}
 }

maxguitextareascintilla.mod/linux_glue.c

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014-2018 Bruce A Henderson
+ Copyright (c) 2014-2019 Bruce A Henderson
  
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
@@ -33,6 +33,7 @@ ScintillaObject * bmx_mgta_scintilla_getsci(void * editor, int id) {
 	scintilla_set_id(obj, id);
 
 	scintilla_send_message(obj, SCI_SETCODEPAGE, SC_CP_UTF8, 0);
+	scintilla_send_message(obj, SCI_ALLOCATELINECHARACTERINDEX, SC_LINECHARACTERINDEX_UTF16 , 0);
 
 	return obj;
 }

maxguitextareascintilla.mod/linuxgtk.bmx

@@ -155,18 +155,14 @@ Type TGTKScintillaTextArea Extends TGTKTextArea
 				startPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos, True)
 				endPos = startPos
 			Else
-				startPos = bmx_mgta_scintilla_bytefromchar(sciPtr, pos, 0, 0)
-				endPos = startPos
+				bmx_mgta_scintilla_startendfromchar(sciPtr, pos, length, startPos, endPos)
 			End If
 		Else
 			If units = TEXTAREA_LINES Then
 				startPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos, True)
 				endPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos + length, True)
 			Else ' must be TEXTAREA_CHARS
-				'startPos = pos
-				'endPos = pos + length
-				startPos = bmx_mgta_scintilla_bytefromchar(sciPtr, pos, 0, 0)
-				endPos = bmx_mgta_scintilla_bytefromchar(sciPtr, length, startPos, pos)
+				bmx_mgta_scintilla_startendfromchar(sciPtr, pos, length, startPos, endPos)
 			End If
 		End If
 
@@ -221,10 +217,7 @@ Type TGTKScintillaTextArea Extends TGTKTextArea
 				startPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos, True)
 				endPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos + length, True)
 			Else ' must be TEXTAREA_CHARS
-				'startPos = pos
-				'endPos = pos + length
-				startPos = bmx_mgta_scintilla_bytefromchar(sciPtr, pos, 0, 0)
-				endPos = bmx_mgta_scintilla_bytefromchar(sciPtr, length, startPos, pos)
+				bmx_mgta_scintilla_startendfromchar(sciPtr, pos, length, startPos, endPos)
 			End If
 
 			bmx_mgta_scintilla_settargetstart(sciPtr, startPos)
@@ -316,8 +309,9 @@ Type TGTKScintillaTextArea Extends TGTKTextArea
 			startPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos, True)
 			realLength = bmx_mgta_scintilla_positionfromline(sciPtr, pos + length, True) - startPos
 		Else ' must be TEXTAREA_CHARS
-			startPos = bmx_mgta_scintilla_bytefromchar(sciPtr, pos, 0, 0)
-			realLength = bmx_mgta_scintilla_bytefromchar(sciPtr, length, startPos, pos) - startPos
+			Local endPos:Int
+			bmx_mgta_scintilla_startendfromchar(sciPtr, pos, length, startPos, endPos)
+			realLength = endPos - startPos
 		End If
 
 		bmx_mgta_scintilla_startstyling(sciPtr, startPos)
@@ -333,10 +327,7 @@ Type TGTKScintillaTextArea Extends TGTKTextArea
 			startPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos, True)
 			endPos = bmx_mgta_scintilla_positionfromline(sciPtr, pos + length, True)
 		Else ' must be TEXTAREA_CHARS
-			'startPos = pos
-			'endPos = pos + length
-			startPos = bmx_mgta_scintilla_bytefromchar(sciPtr, pos, 0, 0)
-			endPos = bmx_mgta_scintilla_bytefromchar(sciPtr, length, startPos, pos)
+			bmx_mgta_scintilla_startendfromchar(sciPtr, pos, length, startPos, endPos)
 		End If
 		
 		Return bmx_mgta_scintilla_gettextrange(sciPtr, startPos, endPos)

maxguitextareascintilla.mod/macos.bmx

@@ -1,4 +1,4 @@
-' Copyright (c) 2014-2018 Bruce A Henderson
+' Copyright (c) 2014-2019 Bruce A Henderson
 ' 
 ' Permission is hereby granted, free of charge, to any person obtaining a copy
 ' of this software and associated documentation files (the "Software"), to deal

maxguitextareascintilla.mod/macos_glue.mm

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014-2018 Bruce A Henderson
+ Copyright (c) 2014-2019 Bruce A Henderson
  
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal

maxguitextareascintilla.mod/maxguitextareascintilla.bmx

@@ -1,4 +1,4 @@
-' Copyright (c) 2014-2018 Bruce A Henderson
+' Copyright (c) 2014-2019 Bruce A Henderson
 ' 
 ' Permission is hereby granted, free of charge, to any person obtaining a copy
 ' of this software and associated documentation files (the "Software"), to deal
@@ -25,10 +25,12 @@ bbdoc: Scintilla-based MaxGUI TextArea gadget.
 End Rem
 Module MaxGUI.MaxGUITextAreaScintilla
 
-ModuleInfo "Version: 1.00"
+ModuleInfo "Version: 1.01"
 ModuleInfo "License: MIT"
-ModuleInfo "Copyright: (c) 2014-2018 Bruce A Henderson"
+ModuleInfo "Copyright: (c) 2014-2019 Bruce A Henderson"
 
+ModuleInfo "History: 1.01"
+ModuleInfo "History: Fixed UTF-8 offset issues."
 ModuleInfo "History: 1.00"
 ModuleInfo "History: Initial Release."
 

maxguitextareascintilla.mod/source.bmx

@@ -1,4 +1,4 @@
-' Copyright (c) 2014-2018 Bruce A Henderson
+' Copyright (c) 2014-2019 Bruce A Henderson
 ' 
 ' Permission is hereby granted, free of charge, to any person obtaining a copy
 ' of this software and associated documentation files (the "Software"), to deal

maxguitextareascintilla.mod/win32.bmx

@@ -1,4 +1,4 @@
-' Copyright (c) 2014-2018 Bruce A Henderson
+' Copyright (c) 2014-2019 Bruce A Henderson
 ' 
 ' Permission is hereby granted, free of charge, to any person obtaining a copy
 ' of this software and associated documentation files (the "Software"), to deal
@@ -162,18 +162,14 @@ Type TWindowsScintillaTextArea Extends TWindowsTextArea
 				startPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos, True)
 				endPos = startPos
 			Else
-				startPos = bmx_mgta_scintilla_bytefromchar(_hwnd, pos, 0, 0)
-				endPos = startPos
+				bmx_mgta_scintilla_startendfromchar(_hwnd, pos, length, startPos, endPos)
 			End If
 		Else
 			If units = TEXTAREA_LINES Then
 				startPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos, True)
 				endPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos + length, True)
 			Else ' must be TEXTAREA_CHARS
-				'startPos = pos
-				'endPos = pos + length
-				startPos = bmx_mgta_scintilla_bytefromchar(_hwnd, pos, 0, 0)
-				endPos = bmx_mgta_scintilla_bytefromchar(_hwnd, length, startPos, pos)
+				bmx_mgta_scintilla_startendfromchar(_hwnd, pos, length, startPos, endPos)
 			End If
 		End If
 
@@ -222,10 +218,7 @@ Type TWindowsScintillaTextArea Extends TWindowsTextArea
 				startPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos, True)
 				endPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos + length, True)
 			Else ' must be TEXTAREA_CHARS
-				'startPos = pos
-				'endPos = pos + length
-				startPos = bmx_mgta_scintilla_bytefromchar(_hwnd, pos, 0, 0)
-				endPos = bmx_mgta_scintilla_bytefromchar(_hwnd, length, startPos, pos)
+				bmx_mgta_scintilla_startendfromchar(_hwnd, pos, length, startPos, endPos)
 			End If
 
 			bmx_mgta_scintilla_settargetstart(_hwnd, startPos)
@@ -317,8 +310,9 @@ Type TWindowsScintillaTextArea Extends TWindowsTextArea
 			startPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos, True)
 			realLength = bmx_mgta_scintilla_positionfromline(_hwnd, pos + length, True) - startPos
 		Else ' must be TEXTAREA_CHARS
-			startPos = bmx_mgta_scintilla_bytefromchar(_hwnd, pos, 0, 0)
-			realLength = bmx_mgta_scintilla_bytefromchar(_hwnd, length, startPos, pos) - startPos
+			Local endPos:Int
+			bmx_mgta_scintilla_startendfromchar(_hwnd, pos, length, startPos, endPos)
+			realLength = endPos - startPos
 		End If
 
 		bmx_mgta_scintilla_startstyling(_hwnd, startPos)
@@ -334,10 +328,7 @@ Type TWindowsScintillaTextArea Extends TWindowsTextArea
 			startPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos, True)
 			endPos = bmx_mgta_scintilla_positionfromline(_hwnd, pos + length, True)
 		Else ' must be TEXTAREA_CHARS
-			'startPos = pos
-			'endPos = pos + length
-			startPos = bmx_mgta_scintilla_bytefromchar(_hwnd, pos, 0, 0)
-			endPos = bmx_mgta_scintilla_bytefromchar(_hwnd, length, startPos, pos)
+			bmx_mgta_scintilla_startendfromchar(_hwnd, pos, length, startPos, endPos)
 		End If
 		
 		Return bmx_mgta_scintilla_gettextrange(_hwnd, startPos, endPos)

maxguitextareascintilla.mod/win32_glue.c

@@ -1,5 +1,5 @@
 /*
- Copyright (c) 2014-2018 Bruce A Henderson
+ Copyright (c) 2014-2019 Bruce A Henderson
  
  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
@@ -42,6 +42,7 @@ HWND bmx_mgta_scintilla_getsci(HWND parent) {
 	scintilla_send_message(obj, SCI_SETCODEPAGE, SC_CP_UTF8, 0);
 	scintilla_send_message(obj, SCI_SETMODEVENTMASK , SC_MODEVENTMASKALL, 0);
 	scintilla_send_message(obj, SCI_SETEOLMODE, SC_EOL_LF, 0); // the default of CRLF results in double LFs for some reason, which breaks (at least) maxide
+	scintilla_send_message(obj, SCI_ALLOCATELINECHARACTERINDEX, SC_LINECHARACTERINDEX_UTF16 , 0);
 
 	return obj;
 }