fixed binary resources (use bas64 encode for most platforms)

moved StringTools.baseEncode/decode to haxe.BaseCode

codegen.ml

@@ -853,3 +853,9 @@ let rec is_volatile t =
 let set_default ctx a c t p =
 	let ve = mk (TLocal a) t p in
 	mk (TIf (mk (TBinop (OpEq,ve,mk (TConst TNull) t p)) ctx.type_api.tbool p, mk (TBinop (OpAssign,ve,mk (TConst c) t p)) t p,None)) ctx.type_api.tvoid p
+
+let bytes_serialize data =
+	let b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789%:" in
+	let tbl = Array.init (String.length b64) (fun i -> String.get b64 i) in
+	let str = Base64.str_encode ~tbl data in
+	"s" ^ string_of_int (String.length str) ^ ":" ^ str

doc/CHANGES.txt

@@ -37,6 +37,8 @@
 	fixed Array.reverse() in PHP
 	fixed Reflect.compareMethods in Neko (require Neko 1.8.0)
 	fixed flash7-8 register usage for __init__ and static variables initialization
+	moved StringTools.baseEncode/Decode to haxe.BaseCode
+	fixed binary resources for Flash, JS and PHP outputs
 
 2008-07-28: 2.0
 	fixed current package bug in inherited constructor type

genjs.ml

@@ -187,9 +187,9 @@ let rec gen_call ctx e el =
 			spr ctx "{ ";
 			spr ctx "name : ";
 			gen_constant ctx e.epos (TString name);
-			spr ctx ", data : [";
-			concat ctx "," (fun s -> gen_constant ctx e.epos (TString s)) (ExtString.String.nsplit data "\000");
-			spr ctx "]}"
+			spr ctx ", data : ";
+			gen_constant ctx e.epos (TString (Codegen.bytes_serialize data));
+			spr ctx "}"
 		) (Hashtbl.fold (fun name data acc -> (name,data) :: acc) ctx.com.resources []);
 		spr ctx "]";
 	| _ ->

genphp.ml

@@ -225,7 +225,7 @@ let write_resource dir name data =
 	let rdir = dir ^ "/res" in
 	if not (Sys.file_exists dir) then Unix.mkdir dir 0o755;
 	if not (Sys.file_exists rdir) then Unix.mkdir rdir 0o755;
-	let ch = open_out (rdir ^ "/" ^ name) in
+	let ch = open_out_bin (rdir ^ "/" ^ name) in
 	output_string ch data;
 	close_out ch
 

genswf8.ml

@@ -892,9 +892,7 @@ and gen_call ctx e el =
 		Hashtbl.iter (fun name data ->
 			incr count;
 			push ctx [VStr ("name",false);VStr (name,true);VStr ("data",false)];
-			let parts = ExtString.String.nsplit data "\000" in
-			List.iter (gen_big_string ctx) (List.rev parts);
-			init_array ctx (List.length parts);
+			gen_big_string ctx (Codegen.bytes_serialize data);
 			push ctx [VInt 2];
 			write ctx AObject;
 			ctx.stack_size <- ctx.stack_size - 4;

genswf9.ml

@@ -1155,7 +1155,7 @@ and gen_call ctx retval e el r =
 			write ctx (HString "name");
 			write ctx (HString name);
 			write ctx (HString "data");
-			write ctx (HString data);
+			write ctx (HString (Codegen.bytes_serialize data));
 			write ctx (HObject 2);
 		) ctx.com.resources;
 		write ctx (HArray !count)

std/StringTools.hx

@@ -220,81 +220,6 @@ class StringTools {
 		#end
 	}
 
-	/**
-		Encode a string using the specified base. The base length must be a power of two.
-	**/
-	public static function baseEncode( s : String, base : String ) : String {
-		#if neko
-			return new String(_baseEncode(untyped s.__s,untyped base.__s));
-		#else
-			var len = base.length;
-			var nbits = 1;
-			while( len > 1 << nbits )
-				nbits++;
-			if( nbits > 8 || len != 1 << nbits )
-				throw "baseEncode: base must be a power of two.";
-			var size = Std.int((s.length * 8 + nbits - 1) / nbits);
-			var out = new StringBuf();
-			var buf = 0;
-			var curbits = 0;
-			var mask = ((1 << nbits) - 1);
-			var pin = 0;
-			while( size-- > 0 ){
-				while( curbits < nbits ){
-					curbits += 8;
-					buf <<= 8;
-					var t = s.charCodeAt(pin++);
-					if( t > 255 )
-						throw "baseEncode: bad chars";
-					buf |= t;
-				}
-				curbits -= nbits;
-				out.addChar(base.charCodeAt((buf >> curbits) & mask));
-			}
-			return out.toString();
-		#end
-	}
-
-	/**
-		Decode a string encoded in the specified base. The base length must be a power of two.
-	**/
-	public static function baseDecode( s : String, base : String ) : String {
-		#if neko
-			return new String(_baseDecode(untyped s.__s,untyped base.__s));
-		#else
-			var len = base.length;
-			var nbits = 1;
-			while( len > 1 << nbits )
-				nbits++;
-			if( nbits > 8 || len != 1 << nbits )
-				throw "baseDecode: base must be a power of two.";
-			var size = (s.length * 8 + nbits - 1) / nbits;
-			var tbl = new Array();
-			for( i in 0...256 )
-				tbl[i] = -1;
-			for( i in 0...len )
-				tbl[base.charCodeAt(i)] = i;
-			var size = (s.length * nbits) / 8;
-			var out = new StringBuf();
-			var buf = 0;
-			var curbits = 0;
-			var pin = 0;
-			while( size-- > 0 ){
-				while( curbits < 8 ){
-					curbits += nbits;
-					buf <<= nbits;
-					var i = tbl[s.charCodeAt(pin++)];
-					if( i == -1 )
-						throw "baseDecode: bad chars";
-					buf |= i;
-				}
-				curbits -= 8;
-				out.addChar((buf >> curbits) & 0xFF);
-			}
-			return out.toString();
-		#end
-	}
-
 	/**
 		Encode a number into a hexadecimal representation, with an optional number of zeros for left padding.
 	**/
@@ -326,8 +251,6 @@ class StringTools {
 	#if neko
 	private static var _urlEncode = neko.Lib.load("std","url_encode",1);
 	private static var _urlDecode = neko.Lib.load("std","url_decode",1);
-	private static var _baseEncode = neko.Lib.load("std","base_encode",2);
-	private static var _baseDecode = neko.Lib.load("std","base_decode",2);
 	#end
 
 }

std/haxe/BaseCode.hx

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2005-2008, The haXe Project Contributors
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   - Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   - Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE HAXE PROJECT CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE HAXE PROJECT CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+package haxe;
+
+class BaseCode {
+
+	var base : haxe.io.Bytes;
+	var nbits : Int;
+	var tbl : Array<Int>;
+
+	public function new( base : haxe.io.Bytes ) {
+		var len = base.length;
+		var nbits = 1;
+		while( len > 1 << nbits )
+			nbits++;
+		if( nbits > 8 || len != 1 << nbits )
+			throw "BaseCode : base length must be a power of two.";
+		this.base = base;
+		this.nbits = nbits;
+	}
+
+	public function encodeBytes( b : haxe.io.Bytes ) : haxe.io.Bytes {
+		#if neko
+		return haxe.io.Bytes.ofData( base_encode(b.getData(),base.getData()) );
+		#else
+		var nbits = this.nbits;
+		var base = this.base;
+		var size = Std.int((b.length * 8 + nbits - 1) / nbits);
+		var out = haxe.io.Bytes.alloc(size);
+		var buf = 0;
+		var curbits = 0;
+		var mask = (1 << nbits) - 1;
+		var pin = 0;
+		var pout = 0;
+		while( pout < size ) {
+			while( curbits < nbits ) {
+				curbits += 8;
+				buf <<= 8;
+				buf |= b.get(pin++);
+			}
+			curbits -= nbits;
+			out.set(pout++,base.get((buf >> curbits) & mask));
+		}
+		return out;
+		#end
+	}
+
+	function initTable() {
+		var tbl = new Array();
+		for( i in 0...256 )
+			tbl[i] = -1;
+		for( i in 0...base.length )
+			tbl[base.get(i)] = i;
+		this.tbl = tbl;
+	}
+
+	public function decodeBytes( b : haxe.io.Bytes ) : haxe.io.Bytes {
+		#if neko
+		return haxe.io.Bytes.ofData( base_decode(b.getData(),base.getData()) );
+		#else
+		var nbits = this.nbits;
+		var base = this.base;
+		if( this.tbl == null ) initTable();
+		var tbl = this.tbl;
+		var size = (b.length * nbits) >> 3;
+		var out = haxe.io.Bytes.alloc(size);
+		var buf = 0;
+		var curbits = 0;
+		var pin = 0;
+		var pout = 0;
+		while( pout < size ) {
+			while( curbits < 8 ) {
+				curbits += nbits;
+				buf <<= nbits;
+				var i = tbl[b.get(pin++)];
+				if( i == -1 )
+					throw "BaseCode : invalid encoded char";
+				buf |= i;
+			}
+			curbits -= 8;
+			out.set(pout++,(buf >> curbits) & 0xFF);
+		}
+		return out;
+		#end
+	}
+
+	public function encodeString( s : String ) {
+		#if neko
+		return neko.NativeString.toString( base_encode(neko.NativeString.ofString(s),base.getData()) );
+		#else
+		return encodeBytes(haxe.io.Bytes.ofString(s)).toString();
+		#end
+	}
+
+	public function decodeString( s : String ) {
+		#if neko
+		return neko.NativeString.toString( base_decode(neko.NativeString.ofString(s),base.getData()) );
+		#else
+		return decodeBytes(haxe.io.Bytes.ofString(s)).toString();
+		#end
+	}
+
+	public static function encode( s : String, base : String ) {
+		var b = new BaseCode(haxe.io.Bytes.ofString(base));
+		return b.encodeString(s);
+	}
+
+	public static function decode( s : String, base : String ) {
+		var b = new BaseCode(haxe.io.Bytes.ofString(base));
+		return b.decodeString(s);
+	}
+
+	#if neko
+	private static var base_encode = neko.Lib.load("std","base_encode",2);
+	private static var base_decode = neko.Lib.load("std","base_decode",2);
+	#end
+}

std/haxe/ImportAll.hx

@@ -46,6 +46,7 @@ import StringTools;
 import Type;
 import Xml;
 
+import haxe.BaseCode;
 import haxe.FastList;
 import haxe.Firebug;
 import haxe.Http;

std/haxe/Resource.hx

@@ -53,7 +53,7 @@ class Resource {
 		return php.io.File.getBytes(getPath(name));
 	}
 #else
-	static var content : Array<{ name : String, data : #if (neko || flash9) String #else Array<String> #end }>;
+	static var content : Array<{ name : String, data : String }>;
 
 	public static function listNames() : Array<String> {
 		var names = new Array();
@@ -62,30 +62,26 @@ class Resource {
 		return names;
 	}
 
-	public static function getString( name : String ) {
+	public static function getString( name : String ) : String {
 		for( x in content )
-			if( x.name == name )
-				return #if neko new String(x.data) #elseif flash9 x.data #else x.data[0] #end;
+			if( x.name == name ) {
+				#if neko
+				return new String(x.data);
+				#else
+				var b : haxe.io.Bytes = haxe.Unserializer.run(x.data);
+				return b.toString();
+				#end
+			}
 		return null;
 	}
 
-	public static function getBytes( name : String ) {
+	public static function getBytes( name : String ) : haxe.io.Bytes {
 		for( x in content )
 			if( x.name == name ) {
 				#if neko
 				return haxe.io.Bytes.ofData(cast x.data);
-				#elseif flash9
-				var b = new flash.utils.ByteArray();
-				b.writeUTFBytes(x.data);
-				return haxe.io.Bytes.ofData(b);
 				#else
-				var buf = new haxe.io.BytesOutput();
-				var first = true;
-				for( seg in x.data ) {
-					if( first ) first = false else buf.writeByte(0);
-					buf.writeString(seg);
-				}
-				return buf.getBytes();
+				return haxe.Unserializer.run(x.data);
 				#end
 			}
 		return null;

std/haxe/Serializer.hx

@@ -246,7 +246,7 @@ class Serializer {
 			case cast haxe.io.Bytes:
 				var v : haxe.io.Bytes = v;
 				#if neko
-				var chars = StringTools.baseEncode(neko.Lib.stringReference(v),BASE64);
+				var chars = new String(base_encode(v.getData(),untyped BASE64.__s));
 				#else
 				var i = 0;
 				var max = v.length - 2;
@@ -387,5 +387,9 @@ class Serializer {
 		return s.toString();
 	}
 
+	#if neko
+	static var base_encode = neko.Lib.load("std","base_encode",2);
+	#end
+
 }
 

std/haxe/Unserializer.hx

@@ -299,8 +299,7 @@ class Unserializer {
  			if( buf.charAt(pos++) != ":" || length - pos < len )
 				throw "Invalid bytes length";
 			#if neko
-			var str = StringTools.baseDecode(buf.substr(pos,len),BASE64);
-			var bytes = neko.Lib.bytesReference(str);
+			var bytes = haxe.io.Bytes.ofData( base_decode(untyped buf.substr(pos,len).__s,untyped BASE64.__s) );
 			#else
 			var codes = CODES;
 			if( codes == null ) {
@@ -347,4 +346,8 @@ class Unserializer {
 		return new Unserializer(v).unserialize();
 	}
 
+	#if neko
+	static var base_decode = neko.Lib.load("std","base_decode",2);
+	#end
+
 }

std/haxe/io/Bytes.hx

@@ -145,9 +145,10 @@ class Bytes {
 		// utf8-encode
 		while( i < max ) {
 			var c = b[i++];
-			if( c < 0x7F )
+			if( c < 0x7F ) {
+				if( c == 0 ) break;
 				s += fcc(c);
-			else if( c < 0xE0 )
+			} else if( c < 0xE0 )
 				s += fcc( ((c & 0x3F) << 6) | (b[i++] & 0x7F) );
 			else if( c < 0xF0 ) {
 				var c2 = b[i++];

tests/unit/TestMisc.hx

@@ -32,6 +32,12 @@ class TestMisc extends Test {
 		allow( haxe.Md5.encode("héllo"), ["1a722f7e6c801d9e470a10cb91ba406d","be50e8478cf24ff3595bc7307fb91b50"] );
 	}
 
+	function testBaseCode() {
+		var b = new haxe.BaseCode(haxe.io.Bytes.ofString("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-"));
+		eq( b.encodeString("Héllow"), "iceFr6NLtM" );
+		eq( b.decodeString("iceFr6NLtM"), "Héllow" );
+	}
+
 	function opt1( ?x : Int, ?y : String ) {
 		return { x : x, y : y };
 	}

tests/unit/TestResource.hx

@@ -18,7 +18,7 @@ class TestResource extends Test {
 			eq( names[1], "res1.txt" );
 		}
 		eq( haxe.Resource.getString("res1.txt"), STR );
-		#if (neko || flash9 || php)
+		#if (neko || php)
 		// allow binary strings
 		eq( haxe.Resource.getString("res2.bin"), "Héllo\000World\000\000\000\000!" );
 		#else