entirely review data file access and optimize DDS loading

hxd/fmt/pak/FileSystem.hx

@@ -55,10 +55,6 @@ private class PakEntry extends FileEntry {
 	var file : Data.File;
 	var pak : FileInput;
 	var subs : Array<PakEntry>;
-
-	var openedBytes : haxe.io.Bytes;
-	var cachedBytes : haxe.io.Bytes;
-	var bytesPosition : Int;
 	var relPath : String;
 
 	public function new(fs, parent, f, p) {
@@ -90,8 +86,6 @@ private class PakEntry extends FileEntry {
 	}
 
 	override function getBytes() {
-		if( cachedBytes != null )
-			return cachedBytes;
 		setPos();
 		fs.totalReadBytes += file.dataSize;
 		fs.totalReadCount++;
@@ -115,40 +109,6 @@ private class PakEntry extends FileEntry {
 		return tot;
 	}
 
-	override function open() {
-		if( openedBytes == null )
-			openedBytes = fs.getCached(this);
-		if( openedBytes == null ) {
-			fs.totalReadBytes += file.dataSize;
-			fs.totalReadCount++;
-			openedBytes = haxe.io.Bytes.alloc(file.dataSize);
-			setPos();
-			pak.readBytes(openedBytes, 0, file.dataSize);
-		}
-		bytesPosition = 0;
-	}
-
-	override function close() {
-		if( openedBytes != null ) {
-			fs.saveCached(this);
-			openedBytes = null;
-		}
-	}
-
-	override function skip( nbytes ) {
-		if( nbytes < 0 || bytesPosition + nbytes > file.dataSize ) throw "Invalid skip";
-		bytesPosition += nbytes;
-	}
-
-	override function readByte() {
-		return openedBytes.get(bytesPosition++);
-	}
-
-	override function read( out : haxe.io.Bytes, pos : Int, len : Int ) {
-		out.blit(pos, openedBytes, bytesPosition, len);
-		bytesPosition += len;
-	}
-
 	override function exists( name : String ) {
 		if( subs != null )
 			for( c in subs )
@@ -203,9 +163,6 @@ class FileSystem implements hxd.fs.FileSystem {
 	var root : PakEntry;
 	var dict : Map<String,PakEntry>;
 	var files : Array<FileInput>;
-	var readCache : Array<PakEntry> = [];
-	var currentCacheSize = 0;
-	public var readCacheSize = 8 << 20; // 8 MB of emulated cache
 	public var totalReadBytes = 0;
 	public var totalReadCount = 0;
 
@@ -243,38 +200,6 @@ class FileSystem implements hxd.fs.FileSystem {
 		files = [];
 	}
 
-	function getCached( e : PakEntry ) {
-		if( readCacheSize == 0 )
-			return null;
-		var index = readCache.lastIndexOf(e);
-		if( index < 0 )
-			return null;
-		if( index != readCache.length - 1 ) {
-			readCache.splice(index, 1);
-			readCache.push(e);
-		}
-		return e.cachedBytes;
-	}
-
-	function saveCached( e : PakEntry ) {
-		if( readCacheSize == 0 )
-			return;
-		var index = readCache.lastIndexOf(e);
-		if( index < 0 ) {
-			// don't cache if too big wrt our size
-			if( e.openedBytes.length > readCacheSize )
-				return;
-			readCache.push(e);
-			e.cachedBytes = e.openedBytes;
-			currentCacheSize += e.cachedBytes.length;
-		}
-		while( currentCacheSize > readCacheSize ) {
-			var e = readCache.shift();
-			currentCacheSize -= e.cachedBytes.length;
-			e.cachedBytes = null;
-		}
-	}
-
 	function addRec( parent : PakEntry, path : String, f : Data.File, pak : FileInput, delta : Int ) {
 		var ent = dict.get(path);
 		if( ent != null ) {

hxd/fs/BytesFileSystem.hx

@@ -6,7 +6,6 @@ class BytesFileEntry extends FileEntry {
 
 	var fullPath : String;
 	var bytes : haxe.io.Bytes;
-	var pos : Int;
 
 	public function new(path, bytes) {
 		this.fullPath = path;
@@ -30,25 +29,6 @@ class BytesFileEntry extends FileEntry {
 		return len;
 	}
 
-	override function open() {
-		pos = 0;
-	}
-
-	override function skip( nbytes : Int ) {
-		pos += nbytes;
-	}
-	override function readByte() : Int {
-		return bytes.get(pos++);
-	}
-
-	override function read( out : haxe.io.Bytes, pos : Int, size : Int ) {
-		out.blit(pos, bytes, this.pos, size);
-		this.pos += size;
-	}
-
-	override function close() {
-	}
-
 	override function load( ?onReady : Void -> Void ) : Void {
 		haxe.Timer.delay(onReady, 1);
 	}

hxd/fs/EmbedFileSystem.hx

@@ -14,7 +14,6 @@ private class EmbedEntry extends FileEntry {
 	#else
 	var data : String;
 	var bytes : haxe.io.Bytes;
-	var readPos : Int;
 	#end
 
 	function new(fs, name, relPath, data) {
@@ -48,54 +47,6 @@ private class EmbedEntry extends FileEntry {
 		return len;
 	}
 
-	override function open() {
-		#if flash
-		if( bytes == null )
-			bytes = Type.createInstance(data, []);
-		bytes.position = 0;
-		#else
-		if( bytes == null ) {
-			bytes = haxe.Resource.getBytes(data);
-			if( bytes == null ) throw "Missing resource " + data;
-		}
-		readPos = 0;
-		#end
-	}
-
-	override function skip( nbytes : Int ) {
-		#if flash
-		bytes.position += nbytes;
-		#else
-		readPos += nbytes;
-		#end
-	}
-
-	override function readByte() : Int {
-		#if flash
-		return bytes.readUnsignedByte();
-		#else
-		return bytes.get(readPos++);
-		#end
-	}
-
-	override function read( out : haxe.io.Bytes, pos : Int, size : Int ) : Void {
-		#if flash
-		bytes.readBytes(out.getData(), pos, size);
-		#else
-		out.blit(pos, bytes, readPos, size);
-		readPos += size;
-		#end
-	}
-
-	override function close() {
-		#if flash
-		bytes = null;
-		#else
-		bytes = null;
-		readPos = 0;
-		#end
-	}
-
 	override function load( ?onReady : Void -> Void ) : Void {
 		#if (flash || js)
 		if( onReady != null ) haxe.Timer.delay(onReady, 1);

hxd/fs/FileEntry.hx

@@ -37,12 +37,7 @@ class FileEntry {
 	}
 
 	public function getText() return getBytes().toString();
-
-	public function open() { }
-	public function skip( nbytes : Int ) { }
-	public function readByte() : Int return 0;
-	public function read( out : haxe.io.Bytes, pos : Int, size : Int ) {}
-	public function close() {}
+	public function open() return @:privateAccess new FileInput(this);
 
 	public function load( ?onReady : Void -> Void ) : Void { if( !isAvailable ) throw "load() not implemented"; else if( onReady != null ) onReady(); }
 	public function loadBitmap( onLoaded : LoadedBitmap -> Void ) : Void { throw "loadBitmap() not implemented"; }

hxd/fs/FileInput.hx

@@ -2,28 +2,89 @@ package hxd.fs;
 
 class FileInput extends haxe.io.Input {
 
-	var f : FileEntry;
+	static var PREFETCH_CACHE : haxe.io.Bytes = null;
 
-	public function new(f) {
-		this.f = f;
-		f.open();
+	var entry : FileEntry;
+	var cache : haxe.io.Bytes;
+	var cachePos : Int;
+	var cacheLen : Int;
+	var nextReadPos : Int;
+
+	function new(entry) {
+		this.entry = entry;
+	}
+
+	public function fetch( dataSize:Int = 256 ) {
+		var prev = cache;
+		if( cache == null || cache.length < dataSize ) {
+			cache = PREFETCH_CACHE;
+			if( cache != null && cache.length >= dataSize ) {
+				PREFETCH_CACHE = null;
+			} else {
+				cache = haxe.io.Bytes.alloc(dataSize);
+			}
+		}
+		var startPos = 0;
+		if( cacheLen > 0 ) {
+			startPos = cacheLen;
+			dataSize -= cacheLen;
+			cache.blit(0, prev, cachePos, cacheLen);
+		}
+		var read = entry.readBytes(cache, startPos, nextReadPos, dataSize);
+		cachePos = 0;
+		cacheLen = startPos + read;
+		nextReadPos += read;
+		if( cacheLen == 0 )
+			throw new haxe.io.Eof();
 	}
 
 	public function skip( nbytes : Int ) {
-		f.skip(nbytes);
+		if( cacheLen > 0 ) {
+			var k = hxd.Math.imin(cacheLen, nbytes);
+			cachePos += k;
+			cacheLen -= k;
+			nbytes -= k;
+		}
+		nextReadPos += nbytes;
 	}
 
 	override function readByte() {
-		return f.readByte();
+		if( cacheLen == 0 ) fetch();
+		var b = cache.get(cachePos++);
+		cacheLen--;
+		return b;
 	}
 
 	override function readBytes( b : haxe.io.Bytes, pos : Int, len : Int ) {
-		f.read(b, pos, len);
-		return len;
+		var tot = 0;
+
+		if( len < 256 && cacheLen < len )
+			fetch();
+
+		if( cacheLen > 0 ) {
+			var k = hxd.Math.imin(len, cacheLen);
+			b.blit(pos, cache, cachePos, k);
+			cachePos += k;
+			cacheLen -= k;
+			len -= k;
+			if( len == 0 )
+				return k;
+			pos += k;
+			tot += k;
+		}
+		if( len > 0 ) {
+			var k = entry.readBytes(b, pos, nextReadPos, len);
+			nextReadPos += k;
+			tot += k;
+		}
+		return tot;
 	}
 
 	override function close() {
-		f.close();
+		if( cache != null && (PREFETCH_CACHE == null || PREFETCH_CACHE.length < cache.length) )
+			PREFETCH_CACHE = cache;
+		cache = null;
+		cacheLen = 0;
 	}
 
 }

hxd/fs/LocalFileSystem.hx

@@ -11,7 +11,6 @@ class LocalEntry extends FileEntry {
 	var relPath : String;
 	var file : String;
 	var originalFile : String;
-	var fread : sys.io.FileInput;
 
 	function new(fs, name, relPath, file) {
 		this.fs = fs;
@@ -27,44 +26,11 @@ class LocalEntry extends FileEntry {
 	override function readBytes( out : haxe.io.Bytes, outPos : Int, pos : Int, len : Int ) : Int {
 		var f = sys.io.File.read(file);
 		f.seek(pos, SeekBegin);
-		var tot = 0;
-		while( len > 0 ) {
-			var k = f.readBytes(out, outPos, len);
-			if( k <= 0 ) break;
-			outPos += k;
-			tot += k;
-			len -= k;
-		}
+		var tot = f.readBytes(out, outPos, len);
 		f.close();
 		return tot;
 	}
 
-	override function open() {
-		if( fread != null )
-			fread.seek(0, SeekBegin);
-		else
-			fread = sys.io.File.read(file);
-	}
-
-	override function skip(nbytes:Int) {
-		fread.seek(nbytes, SeekCur);
-	}
-
-	override function readByte() {
-		return fread.readByte();
-	}
-
-	override function read( out : haxe.io.Bytes, pos : Int, size : Int ) : Void {
-		fread.readFullBytes(out, pos, size);
-	}
-
-	override function close() {
-		if( fread != null ) {
-			fread.close();
-			fread = null;
-		}
-	}
-
 	var isDirCached : Null<Bool>;
 	override function get_isDirectory() : Bool {
 		if( isDirCached != null ) return isDirCached;

hxd/fs/MultiFileSystem.hx

@@ -14,12 +14,7 @@ private class MultiFileEntry extends FileEntry {
 	override function getBytes() : haxe.io.Bytes return el[0].getBytes();
 	override function readBytes( bytes, outPos, pos, len ) return el[0].readBytes(bytes,outPos,pos,len);
 
-	override function open() el[0].open();
-	override function skip( nbytes : Int ) el[0].skip(nbytes);
-
-	override function readByte() : Int return el[0].readByte();
-	override function read( out : haxe.io.Bytes, pos : Int, size : Int ) return el[0].read(out, pos, size);
-	override function close() el[0].close();
+	override function open() return el[0].open();
 
 	override function load( ?onReady : Void -> Void ) : Void return el[0].load(onReady);
 	override function loadBitmap( onLoaded : LoadedBitmap -> Void ) : Void return el[0].loadBitmap(onLoaded);

hxd/res/Gradients.hx

@@ -125,7 +125,7 @@ class Gradients extends Resource {
 
 	function getData() : Data {
 		if (data != null) return data;
-		var fs = new hxd.fs.FileInput(entry);
+		var fs = entry.open();
 		data = new hxd.fmt.grd.Reader(fs).read();
 		fs.close();
 		return data;

hxd/res/Image.hx

@@ -80,7 +80,8 @@ class Image extends Resource {
 		if( inf != null )
 			return inf;
 		inf = new ImageInfo();
-		var f = new hxd.fs.FileInput(entry);
+		var f = entry.open();
+		f.fetch(256); // should be enough to fit DDS header
 		var head = try f.readUInt16() catch( e : haxe.io.Eof ) 0;
 		switch( head ) {
 		case 0xD8FF: // JPG
@@ -128,19 +129,13 @@ class Image extends Resource {
 			inf.height = f.readUInt16();
 
 		case 0x4444: // DDS
-			#if editor
-			var f = new haxe.io.BytesInput(f.read(33*4+10));
-			inline function skip(n) f.position += n;
-			#else
-			inline function skip(n) f.skip(n);
-			#end
 			inf.dataFormat = Dds;
-			skip(10);
+			f.skip(10);
 			inf.height = f.readInt32();
 			inf.width = f.readInt32();
-			skip(2*4);
+			f.skip(2*4);
 			inf.mipLevels = f.readInt32();
-			skip(12*4);
+			f.skip(12*4);
 			var caps = f.readInt32();
 			var fourCC = f.readInt32();
 			var bpp = f.readInt32();
@@ -169,7 +164,7 @@ class Image extends Resource {
 				default: null;
 				}
 			case _ if( fourCC == 0x30315844 /* DX10 */ ):
-				skip(3 * 4);
+				f.skip(3 * 4);
 				inf.flags.set(Dxt10Header);
 				var dxgi = f.readInt32(); // DXGI_FORMAT_xxxx value
 				inf.pixelFormat = switch( dxgi ) {
@@ -183,7 +178,7 @@ class Image extends Resource {
 					throw entry.path+" has unsupported DXGI format "+dxgi;
 				}
 				var imgType = f.readInt32();
-				skip(4);
+				f.skip(4);
 				inf.layerCount = f.readInt32();
 			case 111: // D3DFMT_R16F
 				inf.pixelFormat = R16F;
@@ -366,11 +361,8 @@ class Image extends Resource {
 				bytes = entry.getBytes();
 			} else {
 				var size = hxd.Pixels.calcDataSize(w, h, inf.pixelFormat);
-				entry.open();
-				entry.skip(pos);
 				bytes = haxe.io.Bytes.alloc(size);
-				entry.read(bytes, 0, size);
-				entry.close();
+				entry.readFull(bytes,pos,size);
 				pos = 0;
 			}
 			pixels = new hxd.Pixels(w, h, bytes, inf.pixelFormat, pos);
@@ -468,11 +460,29 @@ class Image extends Resource {
 				// immediately loading the PNG is faster than going through loadBitmap
 				@:privateAccess tex.customMipLevels = inf.mipLevels;
 				tex.alloc();
-				for( layer in 0...tex.layerCount ) {
-					for( mip in 0...inf.mipLevels ) {
-						var pixels = getPixels(tex.format,null,layer * inf.mipLevels + mip);
-						tex.uploadPixels(pixels,mip,layer);
-						pixels.dispose();
+				switch( inf.dataFormat ) {
+				case Dds:
+					var pos = 128;
+					if( inf.flags.has(Dxt10Header) ) pos += 20;
+					for( layer in 0...tex.layerCount ) {
+						for( mip in 0...inf.mipLevels ) {
+							var w = inf.width >> mip;
+							var h = inf.height >> mip;
+							if( w == 0 ) w = 1;
+							if( h == 0 ) h = 1;
+							var size = hxd.Pixels.calcDataSize(w, h, inf.pixelFormat);
+							var bytes = entry.fetchBytes(pos, size);
+							tex.uploadPixels(new hxd.Pixels(w,h,bytes,inf.pixelFormat),mip,layer);
+							pos += size;
+						}
+					}
+				default:
+					for( layer in 0...tex.layerCount ) {
+						for( mip in 0...inf.mipLevels ) {
+							var pixels = getPixels(tex.format,null,layer * inf.mipLevels + mip);
+							tex.uploadPixels(pixels,mip,layer);
+							pixels.dispose();
+						}
 					}
 				}
 				tex.realloc = loadTexture;

hxd/res/Model.hx

@@ -3,7 +3,7 @@ package hxd.res;
 class Model extends Resource {
 
 	public function toHmd() : hxd.fmt.hmd.Library {
-		var fs = new hxd.fs.FileInput(entry);
+		var fs = entry.open();
 		var hmd = new hxd.fmt.hmd.Reader(fs).readHeader(#if editor true #end);
 		fs.close();
 		return new hxd.fmt.hmd.Library(this, hmd);