Deal with number separator in lexer (#10514)

* [lexer] accept `_` as number separator

see #10480

* add some tests

* deal with generators

* well whatever

* allow _f64

src/core/texpr.ml

@@ -566,6 +566,9 @@ let rec constructor_side_effects e =
 		with Exit ->
 			true
 
+let replace_separators s c =
+	String.concat c (ExtString.String.nsplit s "_")
+
 let type_constant basic c p =
 	match c with
 	| Int (s,_) ->

src/generators/gencpp.ml

@@ -28,6 +28,8 @@ open Globals
 *)
 let follow = Abstract.follow_with_abstracts
 
+let replace_float_separators s = Texpr.replace_separators s ""
+
 (*
    Code for generating source files.
    It manages creating diretories, indents, blocks and only modifying files
@@ -1672,7 +1674,7 @@ and cpp_class_path_of klass =
 let cpp_const_type cval = match cval with
    | TInt i -> CppInt(i) , TCppScalar("int")
    | TBool b -> CppBool(b) , TCppScalar("bool")
-   | TFloat f -> CppFloat(f) , TCppScalar("Float")
+   | TFloat f -> CppFloat(replace_float_separators f) , TCppScalar("Float")
    | TString s -> CppString(s) , TCppString
    | _ -> (* TNull, TThis & TSuper should already be handled *)
       CppNull, TCppNull
@@ -1707,7 +1709,7 @@ let rec const_int_of expr =
 let rec const_float_of expr =
    match expr.eexpr with
    | TConst TInt x -> Printf.sprintf "%ld" x
-   | TConst TFloat x -> x
+   | TConst TFloat x -> (replace_float_separators x)
    | TConst TBool x -> if x then "1" else "0"
    | TParenthesis e -> const_float_of e
    | _ -> raise Not_found
@@ -3333,7 +3335,7 @@ let string_of_path path =
 let default_value_string ctx value =
 match value.eexpr with
    | TConst (TInt i) -> Printf.sprintf "%ld" i
-   | TConst (TFloat float_as_string) -> "((Float)" ^ float_as_string ^ ")"
+   | TConst (TFloat float_as_string) -> "((Float)" ^ (replace_float_separators float_as_string) ^ ")"
    | TConst (TString s) -> strq ctx s
    | TConst (TBool b) -> (if b then "true" else "false")
    | TConst TNull -> "null()"
@@ -7439,7 +7441,7 @@ class script_writer ctx filename asciiOut =
       end
    method constText c = match c with
    | TInt i -> (this#op IaConstInt) ^ (Printf.sprintf "%ld " i)
-   | TFloat f -> (this#op IaConstFloat) ^ (this#stringText f)
+   | TFloat f -> (this#op IaConstFloat) ^ (this#stringText (replace_float_separators f))
    | TString s -> (this#op IaConstString) ^ (this#stringText s)
    | TBool true -> (this#op IaConstTrue)
    | TBool false -> (this#op IaConstFalse)

src/generators/gencs.ml

@@ -1369,6 +1369,7 @@ let generate con =
 									| _ -> ()
 								*)
 							| TFloat s ->
+								let s = Texpr.replace_separators s "" in
 								let len = String.length s in
 								let rec loop i prev_c =
 									if i >= len then begin

src/generators/genlua.ml

@@ -56,6 +56,7 @@ type object_store = {
     mutable os_fields : object_store list;
 }
 
+let replace_float_separators s =  Texpr.replace_separators s ""
 
 let debug_expression expression  =
     " --[[ " ^ Type.s_expr_kind expression  ^ " --]] "
@@ -324,7 +325,7 @@ let rec extract_expr e = match e.eexpr with
 
 let gen_constant ctx p = function
     | TInt i -> print ctx "%ld" i
-    | TFloat s -> spr ctx s
+    | TFloat s -> spr ctx (replace_float_separators s)
     | TString s -> begin
             add_feature ctx "use.string";
             print ctx "\"%s\"" (s_escape_lua s)

src/generators/genneko.ml

@@ -172,7 +172,7 @@ let gen_constant ctx pe c =
 		with _ ->
 			if ctx.version < 2 then abort "This integer is too big to be compiled to a Neko 31-bit integer. Please use a Float instead" pe;
 			(EConst (Int32 i),p))
-	| TFloat f -> (EConst (Float f),p)
+	| TFloat f -> (EConst (Float (Texpr.replace_separators f "")),p)
 	| TString s -> call p (field p (ident p "String") "new") [gen_big_string ctx p s]
 	| TBool b -> (EConst (if b then True else False),p)
 	| TNull -> null p

src/syntax/lexer.ml

@@ -111,25 +111,40 @@ let is_valid_identifier s =
 		with Exit ->
 			false
 
+let split_suffix s is_int =
+	let len = String.length s in
+	let rec loop i pivot =
+		if i = len then begin
+			match pivot with
+			| None ->
+				(s,None)
+			| Some pivot ->
+				(* There might be a _ at the end of the literal because we allow _f64 and such *)
+				let literal_length = if String.unsafe_get s (pivot - 1) = '_' then pivot - 1 else pivot in
+				let literal = String.sub s 0 literal_length in
+				let suffix  = String.sub s pivot (len - pivot) in
+				(literal, Some suffix)
+		end else begin
+			let c = String.unsafe_get s i in
+			match c with
+			| 'i' | 'u' ->
+				loop (i + 1) (Some i)
+			| 'f' when not is_int ->
+				loop (i + 1) (Some i)
+			| _ ->
+				loop (i + 1) pivot
+		end
+	in
+	loop 0 None
+
 let split_int_suffix s =
-	let is_signed = String.contains s 'i' in
-	match String.index_opt s (if is_signed then 'i' else 'u') with
-	| Some pivot ->
-		let literal = String.sub s 0 pivot in
-		let suffix  = String.sub s pivot ((String.length s) - pivot) in
-		Const (Int (literal, Some suffix))
-	| None ->
-		Const (Int (s, None))
+	let (literal,suffix) = split_suffix s true in
+	Const (Int (literal,suffix))
 
 let split_float_suffix s =
-	match String.index_opt s 'f' with
-	| Some pivot ->
-		let literal = String.sub s 0 pivot in
-		let suffix  = String.sub s pivot ((String.length s) - pivot) in
-		Const (Float (literal, Some suffix))
-	| None ->
-		Const (Float (s, None))
-	
+	let (literal,suffix) = split_suffix s false in
+	Const (Float (literal,suffix))
+
 let init file =
 	let f = make_file file in
 	cur := f;
@@ -318,11 +333,17 @@ let string_is_whitespace s =
 
 let idtype = [%sedlex.regexp? Star '_', 'A'..'Z', Star ('_' | 'a'..'z' | 'A'..'Z' | '0'..'9')]
 
-let integer = [%sedlex.regexp? ('1'..'9', Star ('0'..'9')) | '0']
+let digit = [%sedlex.regexp? '0'..'9']
+let sep_digit = [%sedlex.regexp? Opt '_', digit]
+let integer_digits = [%sedlex.regexp? (digit, Star sep_digit)]
+let hex_digit = [%sedlex.regexp? '0'..'9'|'a'..'f'|'A'..'F']
+let sep_hex_digit = [%sedlex.regexp? Opt '_', hex_digit]
+let hex_digits = [%sedlex.regexp? (hex_digit, Star sep_hex_digit)]
+let integer = [%sedlex.regexp? ('1'..'9', Star sep_digit) | '0']
 
-let integer_suffix = [%sedlex.regexp? ('i'|'u'), Plus integer]
+let integer_suffix = [%sedlex.regexp? Opt '_', ('i'|'u'), Plus integer]
 
-let float_suffix = [%sedlex.regexp? 'f', Plus integer]
+let float_suffix = [%sedlex.regexp? Opt '_', 'f', Plus integer]
 
 (* https://www.w3.org/TR/xml/#sec-common-syn plus '$' for JSX *)
 let xml_name_start_char = [%sedlex.regexp? '$' | ':' | 'A'..'Z' | '_' | 'a'..'z' | 0xC0 .. 0xD6 | 0xD8 .. 0xF6 | 0xF8 .. 0x2FF | 0x370 .. 0x37D | 0x37F .. 0x1FFF | 0x200C .. 0x200D | 0x2070 .. 0x218F | 0x2C00 .. 0x2FEF | 0x3001 .. 0xD7FF | 0xF900 .. 0xFDCF | 0xFDF0 .. 0xFFFD | 0x10000 .. 0xEFFFF]
@@ -342,16 +363,16 @@ let rec token lexbuf =
 	| Plus (Chars " \t") -> token lexbuf
 	| "\r\n" -> newline lexbuf; token lexbuf
 	| '\n' | '\r' -> newline lexbuf; token lexbuf
-	| "0x", Plus ('0'..'9'|'a'..'f'|'A'..'F'), Opt integer_suffix ->
+	| "0x", Plus hex_digits, Opt integer_suffix ->
 		mk lexbuf (split_int_suffix (lexeme lexbuf))
 	| integer, Opt integer_suffix ->
 		mk lexbuf (split_int_suffix (lexeme lexbuf))
 	| integer, float_suffix ->
 		mk lexbuf (split_float_suffix (lexeme lexbuf))
-	| integer, '.', Plus '0'..'9', Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
-	| '.', Plus '0'..'9', Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
-	| integer, ('e'|'E'), Opt ('+'|'-'), Plus '0'..'9', Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
-	| integer, '.', Star '0'..'9', ('e'|'E'), Opt ('+'|'-'), Plus '0'..'9', Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
+	| integer, '.', Plus integer_digits, Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
+	| '.', Plus integer_digits, Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
+	| integer, ('e'|'E'), Opt ('+'|'-'), Plus integer_digits, Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
+	| integer, '.', Star digit, ('e'|'E'), Opt ('+'|'-'), Plus integer_digits, Opt float_suffix -> mk lexbuf (split_float_suffix (lexeme lexbuf))
 	| integer, "..." ->
 		let s = lexeme lexbuf in
 		mk lexbuf (IntInterval (String.sub s 0 (String.length s - 3)))

tests/unit/src/unit/TestMain.hx

@@ -51,6 +51,7 @@ function main() {
 		new TestOps(),
 		new TestBasetypes(),
 		new TestNumericSuffixes(),
+		new TestNumericSeparator(),
 		new TestExceptions(),
 		new TestBytes(),
 		new TestIO(),

tests/unit/src/unit/TestNumericSeparator.hx

@@ -0,0 +1,75 @@
+package unit;
+
+class TestNumericSeparator extends Test {
+	public function test() {
+		// normal int
+		eq(12_0, 120);
+		eq(1_2_0, 120);
+
+		// hex int
+		eq(0x12_0, 0x120);
+		eq(0x1_2_0, 0x120);
+
+		// normal float
+		feq(12.3_4, 12.34);
+		feq(1_2.34, 12.34);
+		feq(1_2.3_4, 12.34);
+
+		// dot float
+		feq(.3_4, .34);
+		feq(.3_4_5, .345);
+
+		// science float
+		feq(1_2e3_4, 12e34);
+		feq(1_2.3e4_5, 12.3e45);
+
+		// int but actually float
+		feq(1_2f64, 12f64);
+	}
+
+	public function testWithSuffix() {
+		// normal int
+		eq(12_0i32, 120i32);
+		eq(1_2_0i32, 120i32);
+
+		// hex int
+		eq(0x12_0i32, 0x120i32);
+		eq(0x1_2_0i32, 0x120i32);
+
+		// normal float
+		feq(12.3_4f64, 12.34f64);
+		feq(1_2.34f64, 12.34f64);
+		feq(1_2.3_4f64, 12.34f64);
+
+		// dot float
+		feq(.3_4f64, .34f64);
+		feq(.3_4_5f64, .345f64);
+
+		// science float
+		feq(1_2e3_4f64, 12e34f64);
+		feq(1_2.3e4_5f64, 12.3e45f64);
+	}
+
+	public function testJustBeforeSuffix() {
+		// normal int
+		eq(12_0_i32, 120i32);
+		eq(1_2_0_i32, 120i32);
+
+		// hex int
+		eq(0x12_0_i32, 0x120i32);
+		eq(0x1_2_0_i32, 0x120i32);
+
+		// normal float
+		feq(12.3_4_f64, 12.34f64);
+		feq(1_2.34_f64, 12.34f64);
+		feq(1_2.3_4_f64, 12.34f64);
+
+		// dot float
+		feq(.3_4_f64, .34f64);
+		feq(.3_4_5_f64, .345f64);
+
+		// science float
+		feq(1_2e3_4_f64, 12e34f64);
+		feq(1_2.3e4_5_f64, 12.3e45f64);
+	}
+}