[hlmem] refacto for hide integration (#739)

other/haxelib/hlmem/Analyzer.hx

@@ -0,0 +1,108 @@
+package hlmem;
+
+import hlmem.Memory;
+using format.hl.Tools;
+
+// A list of ansi colors is available at
+// https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797#8-16-colors
+enum abstract TextColor(Int) {
+	var Black = 30;
+	var Red = 31;
+	var Green = 32;
+	var Yellow = 33;
+	var Blue = 34;
+	var Magenta = 35;
+	var Cyan = 36;
+	var White = 37;
+}
+
+class Analyzer {
+
+	public static var displayProgress = true;
+	public static var displayFields : FieldsMode = Full;
+	public static var maxLines : Int = 100;
+	public static var useColor : Bool = true;
+
+	public var code : format.hl.Data;
+	var mem : Memory;
+	var otherMems : Array<Memory> = [];
+
+	public function new() {
+	}
+
+	public function loadBytecode( file : String ) {
+		if( code != null ) throw "Duplicate code";
+		code = new format.hl.Reader(false).read(new haxe.io.BytesInput(sys.io.File.getBytes(file)));
+		Analyzer.log(file + " code loaded");
+	}
+
+	public function loadMemoryDump( file : String ) : Memory {
+		var m = new Memory(this);
+		m.load(file);
+		if( mem == null ) {
+			mem = m;
+		} else {
+			otherMems.push(m);
+		}
+		return m;
+	}
+
+	public function build( filter : FilterMode = None ) {
+		mem.build();
+		for( m2 in otherMems ) {
+			m2.buildBlockTypes();
+		}
+		mem.otherMems = [for (i in otherMems) i];
+		mem.filterMode = filter;
+		mem.buildFilteredBlocks();
+	}
+
+	public function nextDump() : Memory {
+		otherMems.push(mem);
+		mem = otherMems.shift();
+		mem.otherMems = [for (i in otherMems) i];
+		mem.build();
+		mem.buildFilteredBlocks();
+		var ostr = otherMems.length > 0 ? (" (others are " + otherMems.map(m -> m.memFile) + ")") : "";
+		Analyzer.log("Using dump " + mem.memFile + ostr);
+		return mem;
+	}
+
+	public inline function getMainMemory() {
+		return mem;
+	}
+
+	public function getMemStats() : Array<hlmem.Result.MemStats> {
+		var objs = [mem.getMemStats()];
+		for( m2 in otherMems ) {
+			objs.push(m2.getMemStats());
+		}
+		return objs;
+	}
+
+	public static function mb( v : Float ) : String {
+		if( v < 1000 )
+			return Std.int(v) + "B";
+		if( v < 1024 * 1000 )
+			return (Math.round(v * 10 / 1024) / 10)+"KB";
+		return (Math.round(v * 10 / (1024 * 1024)) / 10)+"MB";
+	}
+
+	public static inline function logProgress( current : Int, max : Int ) {
+		if( displayProgress && current % 1000 == 0 )
+			Sys.print(Std.int((current * 1000.0 / max) / 10) + "%  \r");
+		if( displayProgress && current == max )
+			Sys.print("       \r");
+	}
+
+	public static inline function log( msg : String ) {
+		Sys.println(msg);
+	}
+
+	public static inline function withColor( str : String, textColor : TextColor ) {
+		if( !useColor )
+			return str;
+		return '\x1B[${textColor}m${str}\x1B[0m';
+	}
+
+}

other/haxelib/hlmem/Block.hx

@@ -24,6 +24,16 @@ abstract Pointer(haxe.Int64) {
 		return haxe.Int64.shr(this,k);
 	}
 
+	@:op(A == B)
+	public static function eq( a : Pointer, b : Pointer ) : Bool {
+		return a.value == b.value;
+	}
+
+	@:op(A != B)
+	public static function neq( a : Pointer, b : Pointer ) : Bool {
+		return a.value != b.value;
+	}
+
 	public static var NULL(get,never) : Pointer;
 	inline static function get_NULL() return new Pointer(0);
 
@@ -41,7 +51,7 @@ class Page {
 	public var kind : PageKind;
 	public var size : Int;
 	public var reserved : Int;
-	public var dataPosition : Int = -1;
+	public var dataPosition : Float;
 
 	public function new() {
 	}
@@ -108,7 +118,7 @@ class Block {
 			parents.push(b);
 		}
 		if( b.subs == null ) b.subs = [];
-		b.subs.push(new BlockSub(this,fid));
+		b.subs.push(new BlockSub(this, fid));
 	}
 
 	public function makeTID( prev : Block, withField : Bool ) {
@@ -161,7 +171,7 @@ class Block {
 		owner = parents[0];
 	}
 
-	function removeParent( p : Block ) {
+	public function removeParent( p : Block ) {
 		if( parents != null ) {
 			parents.remove(p);
 			if( parents.length == 0 ) parents = null;

other/haxelib/hlmem/FileReader.hx

@@ -0,0 +1,111 @@
+package hlmem;
+
+#if js
+enum FileSeekMode {
+	SeekBegin;
+	SeekCur;
+	SeekEnd;
+}
+#else
+typedef FileSeekMode = sys.io.FileSeek;
+#end
+
+class FileReader {
+
+	#if js
+	// js read file in once for better performance
+	var memBytes : haxe.io.Bytes;
+	var memPos : Int;
+	#else
+	var memInput : sys.io.FileInput;
+	#end
+
+	public inline function new(file : String) {
+		#if js
+		memBytes = sys.io.File.getBytes(file);
+		memPos = 0;
+		#else
+		memInput = sys.io.File.read(file);
+		#end
+	}
+
+	public inline function close() {
+		#if js
+		memBytes = null;
+		memPos = 0;
+		#else
+		if( memInput != null )
+			memInput.close();
+		memInput = null;
+		#end
+	}
+
+	public inline function readString(length : Int) : String {
+		#if js
+		var str = memBytes.getString(memPos, 3);
+		memPos += 3;
+		#else
+		var str = memInput.read(3).toString();
+		#end
+		return str;
+	}
+
+	public inline function readByte() : Int  {
+		#if js
+		var b = memBytes.get(memPos);
+		memPos += 1;
+		#else
+		var b = memInput.readByte();
+		#end
+		return b;
+	}
+
+	public inline function readInt32() : Int {
+		#if js
+		var i = memBytes.getInt32(memPos);
+		memPos += 4;
+		#else
+		var i = memInput.readInt32();
+		#end
+		return i;
+	}
+
+	public inline function readPointer( is64 : Bool ) : Block.Pointer {
+		var low = readInt32();
+		var high = is64 ? readInt32() : 0;
+		return cast haxe.Int64.make(high,low);
+	}
+
+	public inline function tell() : Float {
+		#if js
+		return memPos;
+		#else
+		return tell2(@:privateAccess memInput.__f);
+		#end
+	}
+
+	#if (hl && hl_ver >= version("1.12.0"))
+	@:hlNative("std","file_seek2") static function seek2( f : sys.io.File.FileHandle, pos : Float, mode : Int ) : Bool { return false; }
+	@:hlNative("std","file_tell2") static function tell2( f : sys.io.File.FileHandle ) : Float { return 0; }
+	#end
+
+	// pos will be cast to Int64
+	public inline function seek( pos : Float, mode : FileSeekMode ) {
+		#if js
+		if( pos > 0x7FFFFFFF ) throw haxe.io.Error.Custom("seek out of bounds");
+		var dpos = Std.int(pos);
+		switch( mode ) {
+		case SeekBegin:
+			memPos = dpos;
+		case SeekCur:
+			memPos += dpos;
+		case SeekEnd:
+			memPos = memBytes.length + dpos;
+		}
+		#else
+		if( !seek2(@:privateAccess memInput.__f, pos, mode.getIndex()) )
+			throw haxe.io.Error.Custom("seek2 failure()");
+		#end
+	}
+
+}

other/haxelib/hlmem/Main.hx

@@ -0,0 +1,172 @@
+package hlmem;
+
+class Main {
+	static var args : Array<String>;
+	static var analyzer : Analyzer;
+
+	static var sortByCount : Bool = false;
+
+	static function parseArgs( str: String ) {
+		str = StringTools.trim(str);
+		var i = 0;
+		var tok = "";
+		var args = [];
+		var escape = false;
+		while(i != str.length) {
+			var c = str.charAt(i++);
+			if(c == '"') {
+				escape = !escape;
+			}
+			else {
+				if(c == " " && !escape) {
+					if(tok.length > 0) args.push(tok);
+					tok = "";
+				}
+				else
+					tok += c;
+			}
+		}
+		if(tok.length > 0) args.push(tok);
+		return args;
+	}
+
+	static function command() : Bool {
+		Sys.print(Analyzer.withColor("> ", Red));
+		var args = parseArgs(Main.args.length > 0 ? Main.args.shift() : Sys.stdin().readLine());
+		var cmd = args.shift();
+		var mem = analyzer.getMainMemory();
+		switch( cmd ) {
+		case "exit", "quit", "q":
+			return false;
+		case "stats":
+			var res = mem.getMemStats();
+			res.print();
+		case "types":
+			var res = mem.getBlockStatsByType();
+			res.sort(sortByCount);
+			res.print();
+		case "false":
+			var res = mem.getFalsePositives(args.shift());
+			res.print();
+		case "unknown":
+			var res = mem.getUnknown();
+			res.sort(sortByCount);
+			res.print();
+		case "locate":
+			var lt = mem.resolveType(args.shift());
+			if( lt != null ) {
+				var res = mem.locate(lt, Std.parseInt(args.shift()));
+				res.sort(sortByCount);
+				res.print();
+			}
+		case "count":
+			var lt = mem.resolveType(args.shift());
+			if( lt != null ) {
+				var res = mem.count(lt, args);
+				res.sort(sortByCount);
+				res.printWithSum = true;
+				res.print();
+			}
+		case "parents":
+			var lt = mem.resolveType(args.shift());
+			if( lt != null ) {
+				var res = mem.parents(lt);
+				res.sort(sortByCount);
+				res.print();
+			}
+		case "subs":
+			var lt = mem.resolveType(args.shift());
+			if( lt != null ) {
+				var res = mem.subs(lt, Std.parseInt(args.shift()));
+				res.sort(sortByCount);
+				res.print();
+			}
+		case "sort":
+			switch( args.shift() ) {
+			case "mem":
+				sortByCount = false;
+			case "count":
+				sortByCount = true;
+			case mode:
+				Sys.println("Unknown sort mode " + mode);
+			}
+		case "fields":
+			switch( args.shift() ) {
+			case "full":
+				Analyzer.displayFields = Full;
+			case "none":
+				Analyzer.displayFields = None;
+			case "parents":
+				Analyzer.displayFields = Parents;
+			case mode:
+				Sys.println("Unknown fields mode " + mode);
+			}
+		case "filter":
+			switch( args.shift() ) {
+			case "n" | "none":
+				mem.filterMode = None;
+			case "i" | "intersect":
+				mem.filterMode = Intersect;
+			case "u" | "unique":
+				mem.filterMode = Unique;
+			case mode:
+				Sys.println("Unknown filter mode " + mode);
+			}
+			mem.buildFilteredBlocks();
+		case "nextDump":
+			analyzer.nextDump();
+		case "lines":
+			var v = args.shift();
+			if( v != null )
+				Analyzer.maxLines = Std.parseInt(v);
+			Sys.println(Analyzer.maxLines == 0 ? "Lines limit disabled" : Analyzer.maxLines + " maximum lines displayed");
+		case null:
+			// Ignore
+		default:
+			Sys.println("Unknown command " + cmd);
+		}
+		return true;
+	}
+
+	static function main() {
+		analyzer = new Analyzer();
+		args = Sys.args();
+
+		//hl.Gc.dumpMemory(); Sys.command("cp memory.hl test.hl");
+
+		var code = null, memory = null;
+		while( args.length > 0 ) {
+			var arg = args.shift();
+			if( StringTools.endsWith(arg, ".hl") ) {
+				code = arg;
+				analyzer.loadBytecode(arg);
+				continue;
+			}
+			if( arg == "-c" || arg == "--color" )
+				continue;
+			if( arg == "--no-color" ) {
+				Analyzer.useColor = false;
+				continue;
+			}
+			if( arg == "--args" ) {
+				Analyzer.displayProgress = false;
+				break;
+			}
+			if (memory == null) {
+				memory = arg;
+			}
+			analyzer.loadMemoryDump(arg);
+		}
+		if( code != null && memory == null ) {
+			var dir = new haxe.io.Path(code).dir;
+			if( dir == null ) dir = ".";
+			memory = dir+"/hlmemory.dump";
+			if( sys.FileSystem.exists(memory) )
+				analyzer.loadMemoryDump(memory);
+		}
+		analyzer.build();
+
+		while( command() ) {
+		}
+	}
+}

other/haxelib/hlmem/Result.hx

@@ -0,0 +1,157 @@
+package hlmem;
+
+abstract class Result {
+	abstract public function print() : Void;
+}
+
+@:structInit
+class MemStats extends Result {
+	public var memFile : String;
+	public var free : Int;
+	public var used : Int;
+	public var filterUsed : Int;
+	public var totalAllocated : Int;
+	public var gc : Int;
+	public var pagesCount : Int;
+	public var pagesSize : Int;
+	public var rootsCount : Int;
+	public var stackCount : Int;
+	public var typesCount : Int;
+	public var closuresCount : Int;
+	public var blockCount : Int;
+	public var filterMode : Memory.FilterMode;
+	public var filteredBlockCount : Int;
+
+	public function toString() {
+		var str = Analyzer.withColor("--- " + memFile + " ---", Cyan);
+		str += "\n" + pagesCount + " pages, " + Analyzer.mb(pagesSize) + " memory";
+		str += "\n" + rootsCount + " roots, "+ stackCount + " stacks";
+		str += "\n" + typesCount + " types, " + closuresCount + " closures";
+		str += "\n" + blockCount + " live blocks, " + Analyzer.mb(used) + " used, " + Analyzer.mb(free) + " free, "+ Analyzer.mb(gc) + " gc";
+		if( filterMode != None )
+			str += "\n" + filteredBlockCount + " blocks in filter, " + Analyzer.mb(filterUsed) + " used";
+		return str;
+	}
+
+	public function print() {
+		Analyzer.log(this.toString());
+	}
+}
+
+class FalsePositiveStats extends Result {
+	public var falses : Array<TType>;
+	public function new() {
+	}
+	public function add( t : TType ) {
+		falses.push(t);
+	}
+	public function sort() {
+		falses.sort((t1, t2) -> t1.falsePositive - t2.falsePositive);
+	}
+	public function print() {
+		for( f in falses )
+			Analyzer.log(f.falsePositive + " count " + f + " " + f.falsePositiveIndexes + "\n    "+ [for( f in f.memFields ) format.hl.Tools.toString(f.t)]);
+	}
+}
+
+class BlockStatsElement {
+	var mem : Memory;
+	public var tl : Array<Int>;
+	public var count : Int = 0;
+	public var size : Int = 0;
+	public function new( mem : Memory, tl : Array<Int> ) {
+		this.mem = mem;
+		this.tl = tl;
+	}
+	public function add( size : Int ) {
+		this.count ++;
+		this.size += size;
+	}
+	public function getTypes() {
+		var ttypes = [];
+		for( tid in tl )
+			ttypes.push(mem.getTypeById(tid));
+		return ttypes;
+	}
+	public function getNames( withTstr : Bool = true, withId : Bool = true, withField : Bool = true ) {
+		var tpath = [];
+		for( tid in tl )
+			tpath.push(mem.getTypeString(tid, withTstr, withId, withField));
+		return tpath;
+	}
+}
+
+class BlockStats extends Result {
+	var mem : Memory;
+	var byT : Map<String, BlockStatsElement>;
+	public var allT : Array<BlockStatsElement>;
+	public var printWithSum : Bool;
+
+	public function new( mem : Memory ) {
+		this.mem = mem;
+		this.byT = [];
+		this.allT = [];
+		this.printWithSum = false;
+	}
+
+	public function add( t : TType, size : Int ) {
+		return addPath([t == null ? 0 : t.tid], size);
+	}
+
+	public function addPath( tl : Array<Int>, size : Int ) {
+		var key = tl.join(" ");
+		var inf = byT.get(key);
+		if( inf == null ) {
+			inf = new BlockStatsElement(mem, tl);
+			byT.set(key, inf);
+			allT.push(inf);
+		}
+		inf.add(size);
+	}
+
+	public function get( t : TType ) : BlockStatsElement {
+		return getPath([t == null ? 0 : t.tid]);
+	}
+
+	public function getPath( tl : Array<Int> ) : BlockStatsElement {
+		var key = tl.join(" ");
+		return byT.get(key);
+	}
+
+	public function sort( byCount = true , asc = true) {
+		if( byCount )
+			allT.sort(function(i1, i2) return asc ? i1.count - i2.count : i2.count - i1.count);
+		else
+			allT.sort(function(i1, i2) return asc ? i1.size - i2.size : i2.size - i1.size);
+	}
+
+	/**
+	 * Create a copy of the current BlockStats from `pos` to `end`.
+	 * (If call add on the copy, a separate entry will be created.)
+	 */
+	public function slice( pos : Int, ?end : Int ) : BlockStats {
+		var newS = new BlockStats(mem);
+		newS.allT = allT.slice(pos, end);
+		// Do not set byT to reduce memory usage and prevent overwrite parent's elements
+		newS.printWithSum = printWithSum;
+		return newS;
+	}
+
+	public function print() {
+		var totCount = 0;
+		var totSize = 0;
+		var max = Analyzer.maxLines;
+		if( max > 0 && allT.length > max ) {
+			Analyzer.log("<ignore "+(allT.length - max)+" lines>");
+			allT = allT.slice(allT.length - max);
+		}
+		for( i in allT ) {
+			totCount += i.count;
+			totSize += i.size;
+			Analyzer.log(Analyzer.withColor(i.count + " count, " + Analyzer.mb(i.size) + " ", Yellow) + i.getNames().join(Analyzer.withColor(' > ', Cyan)));
+		}
+		if( printWithSum )
+			Analyzer.log("Total: " + totCount + " count, " + Analyzer.mb(totSize));
+	}
+
+}

other/haxelib/hlmem/TType.hx

@@ -175,7 +175,7 @@ class TType {
 	public function toString() {
 		switch( t ) {
 		case HAbstract("roots"):
-			return Memory.withColor("roots", 32);
+			return Analyzer.withColor("roots", Green);
 		case HAbstract(p):
 			return p;
 		case HFun(_), HMethod(_):
@@ -185,4 +185,4 @@ class TType {
 		}
 	}
 
-}
+}

other/haxelib/memory.hxml

@@ -1,3 +1,3 @@
 -lib format
--main hlmem.Memory
+-main hlmem.Main
 -hl memory.hl