add marker chunks, read hxb modules delayed

see #11493

src/compiler/hxb/hxbAbstractReader.ml

@@ -9,7 +9,7 @@ class virtual hxb_abstract_reader = object(self)
 		reader#read (self :> hxb_reader_api) input
 
 	method read_chunks (chunks : cached_chunks) (stats : HxbReader.hxb_reader_stats) =
-		fst (self#read_chunks_until chunks stats LST)
+		fst (self#read_chunks_until chunks stats EOM)
 
 	method read_chunks_until (chunks : cached_chunks) (stats : HxbReader.hxb_reader_stats) (until : HxbData.chunk_kind) =
 		let reader = new HxbReader.hxb_reader stats in

src/compiler/hxb/hxbData.ml

@@ -15,9 +15,10 @@ exception HxbFailure of string
 	EF = enum field
 	AF = anon field
 	EX = expression
-	...F = forward definition
-	...R = reference
-	...D = definition
+	EO = end of (Types | Fields | Module)
+	..F = forward definition
+	..R = reference
+	..D = definition
 *)
 
 type chunk_kind =
@@ -37,6 +38,7 @@ type chunk_kind =
 	| END (* enum definition *)
 	| ABD (* abstract definition *)
 	| TDD (* typedef definition *)
+	| EOT (* end of module types *)
 	(* Field references *)
 	| EFR (* enum field references *)
 	| CFR (* class field references *)
@@ -44,8 +46,9 @@ type chunk_kind =
 	| CFD (* class fields *)
 	| EFD (* enum fields *)
 	| AFD (* abstract fields *)
+	| EOF (* end of fields *)
 	| EXD (* class field expressions *)
-	| LST (* last *)
+	| EOM (* end of module *)
 
 type cached_chunk = chunk_kind * bytes
 type cached_chunks = cached_chunk list
@@ -73,11 +76,13 @@ let string_of_chunk_kind = function
 	| END -> "END"
 	| ABD -> "ABD"
 	| TDD -> "TDD"
+	| EOT -> "EOT"
 	| CFD -> "CFD"
 	| EFD -> "EFD"
 	| AFD -> "AFD"
+	| EOF -> "EOF"
 	| EXD -> "EXD"
-	| LST -> "LST"
+	| EOM -> "EOM"
 
 let chunk_kind_of_string = function
 	| "STR" -> STR
@@ -95,11 +100,13 @@ let chunk_kind_of_string = function
 	| "END" -> END
 	| "ABD" -> ABD
 	| "TDD" -> TDD
+	| "EOT" -> EOT
 	| "CFD" -> CFD
 	| "EFD" -> EFD
 	| "AFD" -> AFD
+	| "EOF" -> EOF
 	| "EXD" -> EXD
-	| "LST" -> LST
+	| "EOM" -> EOM
 	| name -> raise (HxbFailure ("Invalid chunk name: " ^ name))
 
 let error (s : string) =

src/compiler/hxb/hxbReader.ml

@@ -1759,31 +1759,35 @@ class hxb_reader
 
 	method private read_chunk_data (kind : chunk_kind) =
 		match kind with
-		| LST ->
-			incr stats.modules_fully_restored;
 		| STR ->
 			string_pool <- self#read_string_pool;
 		| DOC ->
 			doc_pool <- self#read_string_pool;
 		| MDF ->
 			current_module <- self#read_mdf;
-		| AFR ->
-			self#read_afr;
 		| MTF ->
 			current_module.m_types <- self#read_mtf;
 			api#add_module current_module;
 		| CLR ->
 			self#read_clr;
+		| ENR ->
+			self#read_enr;
 		| ABR ->
 			self#read_abr;
 		| TDR ->
 			self#read_tdr;
-		| ENR ->
-			self#read_enr;
+		| AFR ->
+			self#read_afr;
 		| CLD ->
 			self#read_cld;
+		| END ->
+			self#read_end;
 		| ABD ->
 			self#read_abd;
+		| TDD ->
+			self#read_tdd;
+		| EOT ->
+			()
 		| EFR ->
 			api#enable_field_access;
 			self#read_efr;
@@ -1792,19 +1796,19 @@ class hxb_reader
 			self#read_cfr;
 		| CFD ->
 			self#read_cfd;
-		| AFD ->
-			self#read_afd;
-		| TDD ->
-			self#read_tdd;
-		| END ->
-			self#read_end;
 		| EFD ->
 			self#read_efd;
+		| AFD ->
+			self#read_afd;
+		| EOF ->
+			()
 		| EXD ->
 			self#read_exd;
+		| EOM ->
+			incr stats.modules_fully_restored;
 
 	method read_chunks (new_api : hxb_reader_api) (chunks : cached_chunks) =
-		fst (self#read_chunks_until new_api chunks LST)
+		fst (self#read_chunks_until new_api chunks EOM)
 
 	method read_chunks_until (new_api : hxb_reader_api) (chunks : cached_chunks) end_chunk =
 		api <- new_api;
@@ -1826,14 +1830,16 @@ class hxb_reader
 		let version = IO.read_byte file_ch in
 		if version <> hxb_version then
 			raise (HxbFailure (Printf.sprintf "version mismatch: hxb version %i, reader version %i" version hxb_version));
-		let rec loop () =
-			let (name,size) = self#read_chunk_prefix in
-			let kind = chunk_kind_of_string name in
-			if kind <> LST then begin
+		(fun end_chunk ->
+			let rec loop () =
+				let (name,size) = self#read_chunk_prefix in
+				let kind = chunk_kind_of_string name in
 				self#read_chunk_data kind;
-				loop()
-			end
-		in
-		loop();
-		current_module
+				if kind <> end_chunk then begin
+					loop()
+				end
+			in
+			loop();
+			current_module
+		)
 end

src/compiler/hxb/hxbWriter.ml

@@ -557,7 +557,7 @@ class hxb_writer
 
 	method start_chunk (kind : chunk_kind) =
 		let initial_size = match kind with
-			| LST -> 0
+			| EOT | EOF | EOM -> 0
 			| MDF -> 16
 			| MTF | CLR | END | ABD | ENR | ABR | TDR | EFR | CFR | AFD -> 64
 			| AFR | CLD | TDD | EFD -> 128
@@ -569,7 +569,7 @@ class hxb_writer
 		chunk <- new_chunk
 
 	method start_temporary_chunk : 'a . int -> (Chunk.t -> 'a) -> 'a = fun initial_size ->
-		let new_chunk = Chunk.create LST (* TODO: something else? *) cp initial_size in
+		let new_chunk = Chunk.create EOM (* TODO: something else? *) cp initial_size in
 		let old_chunk = chunk in
 		chunk <- new_chunk;
 		(fun f ->
@@ -2201,7 +2201,9 @@ class hxb_writer
 		Chunk.write_string chunk (Path.UniqueKey.lazy_path m.m_extra.m_file);
 		IOChunk.write_uleb128 chunk.io (DynArray.length anons#items);
 		IOChunk.write_uleb128 chunk.io (DynArray.length tmonos#items);
-		self#start_chunk LST;
+		self#start_chunk EOT;
+		self#start_chunk EOF;
+		self#start_chunk EOM;
 		DynArray.add chunks cp#finalize;
 		if not docs#is_empty then
 			DynArray.add chunks docs#finalize

src/context/display/displayJson.ml

@@ -137,7 +137,7 @@ class hxb_reader_api_com
 			cc#find_module m_path
 		with Not_found ->
 			let mc = cc#get_hxb_module m_path in
-			fst (self#read_chunks_until mc.mc_chunks com.hxb_reader_stats (if headers_only then LST else MTF))
+			fst (self#read_chunks_until mc.mc_chunks com.hxb_reader_stats (if headers_only then EOM else MTF))
 
 	method basic_types =
 		com.basic

src/typing/typeloadModule.ml

@@ -812,7 +812,13 @@ let rec get_reader ctx p =
 and load_hxb_module ctx path p =
 	let read file input =
 		try
-			(get_reader ctx p)#read_hxb input ctx.com.hxb_reader_stats
+			let read = (get_reader ctx p)#read_hxb input ctx.com.hxb_reader_stats in
+			let m = read EOT in
+			delay ctx PBuildClass (fun () ->
+				ignore(read EOF);
+				delay ctx PTypeField (fun () -> ignore(read EOM));
+			);
+			m
 		with e ->
 			Printf.eprintf "\x1b[30;41mError loading %s from %s\x1b[0m\n" (snd path) file;
 			let msg = Printexc.to_string e and stack = Printexc.get_backtrace () in

tests/unit/compile-hxb-jvm-read.hxml

@@ -0,0 +1,20 @@
+-D source-header=
+--debug
+# -p src
+# -cp "C:\Program Files\The Haxe Effect\src/dev/null"
+--resource res1.txt@re/s?!%[]))("'1.txt
+--resource res2.bin@re/s?!%[]))("'1.bin
+--resource serializedValues.txt
+--macro Macro.init()
+--dce full
+-lib utest
+-D analyzer-optimize
+-D analyzer-user-var-fusion
+-D message.reporting=pretty
+-D haxe-next
+
+--main unit.TestMain
+--java-lib native_java/native.jar
+--jvm bin/unit.jar
+-D jvm.compression-level=0
+--hxb-lib bin/hxb/jvm.zip

tests/unit/compile-jvm-only.hxml

@@ -2,4 +2,5 @@ compile-each.hxml
 --main unit.TestMain
 --java-lib native_java/native.jar
 --jvm bin/unit.jar
--D jvm.compression-level=0
+-D jvm.compression-level=0
+--hxb bin/hxb/jvm.zip