Sqlite specification, fixes and tests (#9703)

* added a test for sys.db.Sqlite

* added a test to check correct file is created for Sqlite

* drop test table on teardown

* documented ResultSet

* fix neko

* Sqlite tests

* adjust php behavior

* fix tests names

* fix cpp

* fix hl

* disable getFIeldsNames test for cpp

* disable Sqlite tests for some targets

* use in-memory db except for the #9700 test

* disable tests for eval & php(windows)

std/cpp/_std/sys/db/Sqlite.hx

@@ -85,12 +85,10 @@ private class SqliteConnection implements Connection {
 
 	public function commit() {
 		request("COMMIT");
-		startTransaction(); // match mysql usage
 	}
 
 	public function rollback() {
 		request("ROLLBACK");
-		startTransaction(); // match mysql usage
 	}
 
 	@:native("_hx_sqlite_connect")
@@ -175,7 +173,7 @@ private class SqliteResultSet implements ResultSet {
 	}
 
 	public function getFieldsNames():Array<String> {
-		return null;
+		throw new haxe.exceptions.NotImplementedException();
 	}
 
 	@:native("_hx_sqlite_result_next")

std/hl/_std/sys/db/Sqlite.hx

@@ -137,12 +137,10 @@ private class SqliteConnection implements Connection {
 
 	public function commit():Void {
 		request("COMMIT");
-		startTransaction(); // match mysql usage
 	}
 
 	public function rollback():Void {
 		request("ROLLBACK");
-		startTransaction(); // match mysql usage
 	}
 }
 

std/neko/_std/sys/db/Sqlite.hx

@@ -75,12 +75,10 @@ private class SqliteConnection implements Connection {
 
 	public function commit() {
 		request("COMMIT");
-		startTransaction(); // match mysql usage
 	}
 
 	public function rollback() {
 		request("ROLLBACK");
-		startTransaction(); // match mysql usage
 	}
 
 	static var _encode = neko.Lib.load("std", "base_encode", 2);
@@ -177,6 +175,12 @@ private class SqliteResultSet implements ResultSet {
 	}
 
 	public function getFieldsNames():Array<String> {
+		if(hasNext()) {
+			return switch cache.first() {
+				case null: null;
+				case row: Reflect.fields(row);
+			}
+		}
 		return null;
 	}
 

std/php/_std/sys/db/Sqlite.hx

@@ -23,6 +23,7 @@
 package sys.db;
 
 import php.*;
+import php.Global.*;
 import php.db.*;
 import sys.db.*;
 
@@ -94,14 +95,9 @@ private class SQLiteResultSet implements ResultSet {
 	public var length(get, null):Int;
 	public var nfields(get, null):Int;
 
-	var _length:Int = 0;
-	var _nfields:Int = 0;
-
-	var loaded:Bool = false;
-	var currentIndex:Int = 0;
-	var rows:NativeIndexedArray<NativeAssocArray<Scalar>>;
+	var cache = new NativeIndexedArray<{}>();
 	var result:SQLite3Result;
-	var fetchedRow:NativeArray;
+	var resultIsDepleted = false;
 	var fieldsInfo:NativeAssocArray<Int>;
 
 	public function new(result:SQLite3Result) {
@@ -109,40 +105,63 @@ private class SQLiteResultSet implements ResultSet {
 	}
 
 	public function hasNext():Bool {
-		if (!loaded)
-			load();
-		return currentIndex < _length;
+		return switch next() {
+			case null: false;
+			case row:
+				array_unshift(cache, row);
+				row;
+		}
 	}
 
 	public function next():Dynamic {
-		if (!loaded)
-			load();
-		var next:Dynamic = rows[currentIndex++];
-		return Boot.createAnon(correctArrayTypes(next));
+		return switch array_shift(cache) {
+			case null: fetchNext();
+			case row: row;
+		}
+	}
+
+	function fetchNext():Null<{}> {
+		return resultIsDepleted ? null : switch result.fetchArray(Const.SQLITE3_ASSOC) {
+			case false:
+				resultIsDepleted = true;
+				result.finalize();
+				null;
+			case row:
+				Boot.createAnon(correctArrayTypes(row));
+		}
+	}
+
+	public function cacheAll():NativeIndexedArray<{}> {
+		var row = fetchNext();
+		while(row != null) {
+			cache.push(row);
+			row = fetchNext();
+		}
+		return cache;
 	}
 
 	public function results():List<Dynamic> {
-		if (!loaded)
-			load();
 		var list = new List();
-		Syntax.foreach(rows, function(_, row) list.add(Boot.createAnon(correctArrayTypes(row))));
+		for(row in cacheAll()) {
+			list.add(row);
+		}
 		return list;
 	}
 
+	function getColumn(n:Int):Any {
+		return array_values(Syntax.array(current()))[n];
+	}
+
 	public function getResult(n:Int):String {
-		if (!loaded)
-			load();
-		if (!hasNext())
-			return null;
-		return Global.array_values(rows[currentIndex])[n];
+		return Syntax.string(getColumn(n));
 	}
 
 	public function getIntResult(n:Int):Int {
-		return Syntax.int(getResult(n));
+		return Syntax.int(getColumn(n));
 	}
 
 	public function getFloatResult(n:Int):Float {
-		return Syntax.float(getResult(n));
+		return Syntax.float(getColumn(n));
 	}
 
 	public function getFieldsNames():Null<Array<String>> {
@@ -150,6 +169,19 @@ private class SQLiteResultSet implements ResultSet {
 		return Global.array_keys(fieldsInfo);
 	}
 
+	function current():Null<{}> {
+		return switch reset(cache) {
+			case false:
+				switch next() {
+					case null: null;
+					case row:
+						cache.push(row);
+						row;
+				}
+			case row: row;
+		}
+	}
+
 	function correctArrayTypes(row:NativeAssocArray<String>):NativeAssocArray<Scalar> {
 		var fieldsInfo = getFieldsInfo();
 		Syntax.foreach(row, function(field:String, value:String) {
@@ -168,13 +200,6 @@ private class SQLiteResultSet implements ResultSet {
 		return fieldsInfo;
 	}
 
-	function load() {
-		loaded = true;
-		_nfields = result.numColumns();
-		getFieldsInfo();
-		fetchAll();
-	}
-
 	function correctType(value:String, type:Int):Scalar {
 		if (value == null)
 			return null;
@@ -185,21 +210,9 @@ private class SQLiteResultSet implements ResultSet {
 		return value;
 	}
 
-	function fetchAll() {
-		rows = Syntax.arrayDecl();
-		var index = 0;
-		var row = result.fetchArray(Const.SQLITE3_ASSOC);
-		while (row != false) {
-			rows[index] = correctArrayTypes(row);
-			row = result.fetchArray(Const.SQLITE3_ASSOC);
-			index++;
-		}
-		_length = index;
-	}
-
-	function get_length()
-		return _length;
+	function get_length():Int
+		return count(cacheAll());
 
-	function get_nfields()
-		return _nfields;
+	function get_nfields():Int
+		return result.numColumns();
 }

std/sys/db/ResultSet.hx

@@ -23,14 +23,48 @@
 package sys.db;
 
 interface ResultSet {
+	/**
+		Get amount of rows left in this set.
+		Depending on a database management system accessing this field may cause
+		all rows to be fetched internally. However, it does not affect `next` calls.
+	**/
 	var length(get, null):Int;
+	/**
+		Amount of columns in a row.
+		Depending on a database management system may return `0` if the query
+		did not match any rows.
+	**/
 	var nfields(get, null):Int;
 
+	/**
+		Tells whether there is a row to be fetched.
+	**/
 	function hasNext():Bool;
+	/**
+		Fetch next row.
+	**/
 	function next():Dynamic;
+	/**
+		Fetch all the rows not fetched yet.
+	**/
 	function results():List<Dynamic>;
+	/**
+		Get the value of `n`-th column of the current row.
+		Throws an exception if the re
+	**/
 	function getResult(n:Int):String;
+	/**
+		Get the value of `n`-th column of the current row as an integer value.
+	**/
 	function getIntResult(n:Int):Int;
+	/**
+		Get the value of `n`-th column of the current row as a float value.
+	**/
 	function getFloatResult(n:Int):Float;
+	/**
+		Get the list of column names.
+		Depending on a database management system may return `null` if there's no
+		more rows to fetch.
+	**/
 	function getFieldsNames():Null<Array<String>>;
 }

tests/sys/src/Main.hx

@@ -11,6 +11,15 @@ class Main {
 		runner.addCase(new io.TestFile());
 		runner.addCase(new io.TestFileInput());
 		runner.addCase(new io.TestProcess());
+		#if !(java || cs || lua || python || eval) // Sqlite is not implemented for these targets
+		#if !hl // Idk how to resolve "FATAL ERROR : Failed to load library sqlite.hdll"
+		var testSqlite = #if php Sys.systemName() != 'Windows' #else true #end; //our CI doesn't have sqlite php module
+		if(testSqlite) {
+			runner.addCase(new db.TestSqliteConnection());
+			runner.addCase(new db.TestSqliteResultSet());
+		}
+		#end
+		#end
 		#if php
 		switch (Sys.systemName()) {
 			case "Windows":

tests/sys/src/db/SqliteSetup.hx

@@ -0,0 +1,27 @@
+package db;
+
+import sys.db.Sqlite;
+import sys.db.Connection;
+import sys.FileSystem;
+
+class SqliteSetup extends utest.Test {
+	var cnx:Connection;
+
+	function setup() {
+		openConnection();
+		cnx.request("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, num REAL, value TEXT)");
+	}
+
+	function teardown() {
+		cnx.request("DROP TABLE test");
+		closeConnection();
+	}
+
+	function closeConnection() {
+		cnx.close();
+	}
+
+	function openConnection() {
+		cnx = Sqlite.open(':memory:');
+	}
+}

tests/sys/src/db/TestSqliteConnection.hx

@@ -0,0 +1,60 @@
+package db;
+
+import utest.Assert.*;
+import sys.FileSystem;
+
+class TestSqliteConnection extends SqliteSetup {
+	function testDbName() {
+		equals('SQLite', cnx.dbName());
+	}
+
+	function testLastInsertId() {
+		cnx.request('INSERT INTO test (num, value) VALUES (10, "one")');
+		equals(1, cnx.lastInsertId());
+
+		cnx.request('INSERT INTO test (num, value) VALUES (20, "two")');
+		equals(2, cnx.lastInsertId());
+	}
+
+	function testTransactionRollback() {
+		cnx.startTransaction();
+		cnx.request('INSERT INTO test (id, num, value) VALUES (1, 2, "one")');
+		cnx.rollback();
+		var result = cnx.request('SELECT * FROM test WHERE id = 1');
+		isFalse(result.hasNext());
+	}
+
+	function testTransactionCommit() {
+		cnx.startTransaction();
+		cnx.request('INSERT INTO test (id, num, value) VALUES (1, 2, "one")');
+		cnx.commit();
+		var result = cnx.request('SELECT * FROM test WHERE id = 1');
+		same({id:1, num:2, value:'one'}, result.next());
+	}
+
+	function testIssue9700() {
+		var dbFile = 'temp/db.sqlite';
+		try sys.FileSystem.deleteFile(dbFile) catch(_) {}
+		var cnx = sys.db.Sqlite.open(dbFile);
+		cnx.request("CREATE TABLE test (id INTEGER)");
+
+		function checkDataSaved(id:Int) {
+			cnx.request('INSERT INTO test (id) VALUES ($id)');
+			cnx.close();
+			cnx = sys.db.Sqlite.open(dbFile);
+			var result = cnx.request('SELECT * FROM test WHERE id = $id');
+			same({id:id}, result.next());
+		}
+
+		cnx.startTransaction();
+		cnx.commit();
+		checkDataSaved(1);
+
+		cnx.startTransaction();
+		cnx.rollback();
+		checkDataSaved(2);
+
+		cnx.request('DROP TABLE test');
+		cnx.close();
+	}
+}

tests/sys/src/db/TestSqliteResultSet.hx

@@ -0,0 +1,62 @@
+package db;
+
+import utest.Assert.*;
+import haxe.ds.ReadOnlyArray;
+
+class TestSqliteResultSet extends SqliteSetup {
+	var data:ReadOnlyArray<{id:Int, num:Float, value:String}> = [
+		{id:1, num:1.5, value:'one'},
+		{id:10, num:10.1, value:'ten'},
+	];
+
+	override function setup() {
+		super.setup();
+		var values = data.map(r -> '(${r.id}, ${r.num}, "${r.value}")').join(',');
+		cnx.request('INSERT INTO test (id, num, value) VALUES $values');
+	}
+
+	function testLength() {
+		var result = cnx.request('SELECT * FROM test');
+		var length = result.length;
+
+		result.next();
+		equals(length - 1, result.length);
+
+		for(_ in result) {}
+		equals(0, result.length);
+	}
+
+	function testNfields() {
+		var result = cnx.request('SELECT * FROM test');
+		equals(3, result.nfields);
+	}
+
+	function testResults() {
+		var result = cnx.request('SELECT * FROM test');
+		same(data, Lambda.array(result.results()));
+	}
+
+#if !cpp //ResultSet.getFieldsNames() is not implemented in cpp.
+
+	function testGetFieldsNames() {
+		var result = cnx.request('SELECT * FROM test ORDER BY id');
+		same(['id', 'num', 'value'], result.getFieldsNames());
+	}
+#end
+
+	function testAsIterator() {
+		var result = cnx.request('SELECT * FROM test ORDER BY id');
+		var rows = [];
+		var expected = data.copy();
+		for(row in result) {
+			same(expected.shift(), row);
+		}
+	}
+
+	function testGetResult() {
+		var result = cnx.request('SELECT * FROM test ORDER BY id');
+		equals('1', result.getResult(0));
+		equals(1, result.getIntResult(0));
+		equals(1, result.getFloatResult(0));
+	}
+}